Compare commits
73 Commits
da2a23f6da
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 874e15c01e | |||
| c7ebef7cc1 | |||
| 27cfa6d68e | |||
| b8ea7374e4 | |||
| 95a368163a | |||
| 52f801675e | |||
| dd63dc97d5 | |||
| c8ffab2c06 | |||
| 3fa76cc74b | |||
| 3859d9d555 | |||
| 2c3ae1de22 | |||
| dd4744df05 | |||
| 1c63ec231f | |||
| e59b0fd6b6 | |||
| 1e7576e7d0 | |||
| e3295afe73 | |||
| 120f52a29a | |||
| 0e53da09c7 | |||
| 111008a858 | |||
| 6322786fd2 | |||
| 0f0aa1021a | |||
| d092abd91a | |||
| a7b1887274 | |||
| b020665088 | |||
| 5ae9763f9a | |||
| 50ba02e79e | |||
| 4e4632ab3d | |||
| 86975e2e2b | |||
| 03cf4b9281 | |||
| 2d232941da | |||
| e325981b17 | |||
| 07539b683e | |||
| 8713169e84 | |||
| 2a92c4e71b | |||
| cc4f3b7dcf | |||
| dc11ac75d2 | |||
| b444009593 | |||
| 18743ffdde | |||
| 064fc43fe3 | |||
| 616769e805 | |||
| 55a21e981f | |||
| fff2b6a52a | |||
| 955e489977 | |||
| f1a3bb9f00 | |||
| 5c2e15e28b | |||
| 474a618f79 | |||
| 9ae0f271ea | |||
| 17b870e4cc | |||
| 99cef94153 | |||
| e2db719fd7 | |||
| b80ed4b483 | |||
| 8022c77b70 | |||
| e10e226743 | |||
| 8c8f8a261c | |||
| 2bdcfe0bae | |||
| 4f9c0d0b75 | |||
| 5180d811a8 | |||
| f1eb453a19 | |||
| 9ee8de8578 | |||
| 1db6475f83 | |||
| ae3b114ebc | |||
| 908c67a26c | |||
| a8158566ac | |||
| 3a8f89432a | |||
| b07c30da5e | |||
| 77ffb0ab2a | |||
| 7c1ceafc6b | |||
| f3d176116a | |||
| a9f2f88604 | |||
| 855976e2b6 | |||
| e66bf2638b | |||
| 3eb09bd861 | |||
| b2ceac1594 |
+6
-4
@@ -1,11 +1,13 @@
|
||||
cert.pem
|
||||
*.db
|
||||
*.db-journal
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
uploads/*.jpg
|
||||
uploads/*.png
|
||||
key.pem
|
||||
cert.pem
|
||||
*.pre-msfs-import
|
||||
__pycache__/
|
||||
.venv/
|
||||
uploads/*.jpg
|
||||
uploads/photos/
|
||||
uploads/*.png
|
||||
.venv/
|
||||
*.bak
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
# Canteen Asset Tracker — Agent Guide
|
||||
# Canteen Asset Tracker
|
||||
|
||||
Mobile-friendly webapp for tracking physical assets with barcode scanning, OCR sticker reading, and GPS check-ins. The main technician-facing application in the Canteen stack.
|
||||
Mobile-friendly webapp for tracking physical assets with barcode scanning, OCR sticker reading, and GPS check-ins. The main technician-facing application.
|
||||
|
||||
...
|
||||
|
||||
## Company & Place Data
|
||||
|
||||
Asset company, customer_name, and place fields are populated from the MSFS Dynamics 365 extraction DB. The migration script `scripts/populate_asset_company_place.py` maps each asset's `connect_id` → `msdyn_customerasset` → `account` table to get:
|
||||
|
||||
- **company** = account name (e.g. "Jeremy B - 1960 Broadway")
|
||||
- **place** = city name (e.g. "Orlando")
|
||||
- **customer_name** = account name (same as company, for search/filter)
|
||||
|
||||
5,743/7,488 assets (~77%) have company data; remaining 1,745 have connect_ids with no account linkage in the extraction DB.
|
||||
|
||||
## Stack
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -23,6 +23,134 @@ from typing import Optional, Tuple
|
||||
DB_PATH = str(Path(__file__).parent / "assets.db")
|
||||
|
||||
|
||||
# ─── Universal identifier normalization (for photo→DB matching) ───────────
|
||||
|
||||
def normalize_identifier(raw: str) -> str:
|
||||
"""
|
||||
Normalize any asset identifier (serial number, barcode, equipment ID,
|
||||
connect ID, machine ID) for comparison.
|
||||
|
||||
- Strips leading label prefixes (S/N:, ID#, Machine ID:, Monyx ID, etc.)
|
||||
- Removes dots, dashes, spaces, slashes, colons
|
||||
- Uppercases
|
||||
- Returns just the alphanumeric core for matching
|
||||
|
||||
Examples:
|
||||
'2500.0100.0025534' → '2500010000255534'
|
||||
'201037BA00039' → '201037BA00039'
|
||||
'S/N: 2500.0100.0025534' → '2500010000255534'
|
||||
'ID# 4434331624226353' → '4434331624226353'
|
||||
'Monyx ID 48602143' → '48602143'
|
||||
'RY10006338' → 'RY10006338'
|
||||
'201037BA00039' → '201037BA00039'
|
||||
"""
|
||||
if not raw:
|
||||
return ''
|
||||
s = raw.strip().upper()
|
||||
# Strip common label prefixes
|
||||
s = re.sub(
|
||||
r'^(S/N|SN|SERIAL|SERIAL\s*NO|ID|UID|MACHINE\s*ID|MACHINE|'
|
||||
r'EQUIPMENT\s*ID|EQ\s*ID|ASSET\s*ID|ITEM|MODEL|PART\s*NO|'
|
||||
r'MONYX\s*ID|PROPERTY\s*OF|BARCODE)\s*[:=#]\s*',
|
||||
'', s, flags=re.IGNORECASE
|
||||
)
|
||||
# Strip leading non-alphanumeric (leftover label debris)
|
||||
s = re.sub(r'^[^A-Z0-9]+', '', s)
|
||||
# Remove all non-alphanumeric (dots, dashes, spaces, etc.)
|
||||
s = re.sub(r'[^A-Z0-9]', '', s)
|
||||
return s
|
||||
|
||||
|
||||
def find_asset_by_normalized_id(db_path: str, normalized: str) -> list:
|
||||
"""
|
||||
Search assets.db for any asset whose serial_number, machine_id, connect_id,
|
||||
equipment_id, or barcode matches the given normalized identifier.
|
||||
|
||||
Supports:
|
||||
- Exact match: normalized string equals the DB field (after normalization)
|
||||
- Suffix match: if the normalized value is all digits and >= 7 chars,
|
||||
match against the last N digits of connect_id and equipment_id (for
|
||||
partial OCR reads that capture only the numeric tail of a longer ID).
|
||||
|
||||
Returns a list of matching rows (dicts).
|
||||
"""
|
||||
if not normalized or len(normalized) < 3:
|
||||
return []
|
||||
|
||||
import sqlite3
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Suffix matching: if the scanned text is purely numeric and >= 7 chars,
|
||||
# search for assets whose connect_id or equipment_id ends with those digits.
|
||||
# Connect IDs often look like VM-05064-0000099387; OCR might only read "0000099387".
|
||||
suffix_conditions = ''
|
||||
suffix_params = []
|
||||
if len(normalized) >= 7 and normalized.isdigit():
|
||||
# Match against the last N digits of connect_id and equipment_id.
|
||||
# Use LIKE with rightmost-anchored pattern: %<digits>
|
||||
suffix_conditions = """
|
||||
OR replace(replace(replace(replace(upper(connect_id), '-', ''), '.', ''), ' ', ''), '/', '') LIKE ?
|
||||
OR replace(replace(replace(replace(upper(equipment_id), '-', ''), '.', ''), ' ', ''), '/', '') LIKE ?
|
||||
"""
|
||||
suffix_params = ['%' + normalized, '%' + normalized]
|
||||
|
||||
rows = conn.execute(f"""
|
||||
SELECT id, machine_id, name, serial_number, connect_id, equipment_id,
|
||||
barcode, make, model, category
|
||||
FROM assets
|
||||
WHERE replace(replace(replace(replace(upper(serial_number), '-', ''), '.', ''), ' ', ''), '/', '') = ?
|
||||
OR replace(replace(replace(replace(upper(connect_id), '-', ''), '.', ''), ' ', ''), '/', '') = ?
|
||||
OR replace(replace(replace(replace(upper(equipment_id), '-', ''), '.', ''), ' ', ''), '/', '') = ?
|
||||
OR replace(replace(replace(replace(upper(machine_id), '-', ''), '.', ''), ' ', ''), '/', '') = ?
|
||||
OR replace(replace(replace(replace(upper(barcode), '-', ''), '.', ''), ' ', ''), '/', '') = ?
|
||||
{suffix_conditions}
|
||||
""", (normalized, normalized, normalized, normalized, normalized, *suffix_params)).fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def find_assets_by_scanned_text(db_path: str, raw_text: str) -> list:
|
||||
"""
|
||||
Given raw OCR text from a label photo, extract all plausible identifiers
|
||||
and search the DB for matches. Returns list of (normalized, field, asset) tuples.
|
||||
"""
|
||||
if not raw_text:
|
||||
return []
|
||||
|
||||
results = []
|
||||
|
||||
# 1. Try each line as a potential identifier
|
||||
lines = raw_text.strip().split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or len(line) < 4:
|
||||
continue
|
||||
norm = normalize_identifier(line)
|
||||
if len(norm) >= 4:
|
||||
matches = find_asset_by_normalized_id(db_path, norm)
|
||||
for m in matches:
|
||||
results.append((norm, line.strip(), m))
|
||||
|
||||
# 2. Also try individual number-like tokens on each line (space-separated values on a line)
|
||||
for line in lines:
|
||||
tokens = re.findall(r'[A-Z0-9]{4,}', line.upper())
|
||||
for token in tokens:
|
||||
if len(token) >= 4:
|
||||
matches = find_asset_by_normalized_id(db_path, token)
|
||||
for m in matches:
|
||||
results.append((token, line.strip(), m))
|
||||
|
||||
# Deduplicate by asset id
|
||||
seen = set()
|
||||
unique = []
|
||||
for norm, src, asset in results:
|
||||
if asset['id'] not in seen:
|
||||
seen.add(asset['id'])
|
||||
unique.append({'normalized': norm, 'source_text': src, 'asset': asset})
|
||||
return unique
|
||||
|
||||
|
||||
# ─── Character substitution (human entry errors) ──────────────────────────
|
||||
|
||||
def normalise_serial(sn: str) -> str:
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Photo Cross-Reference Report
|
||||
**Generated:** 2026-05-29
|
||||
|
||||
## Database
|
||||
- assets.db: 7,488 assets
|
||||
|
||||
## Photos Scanned
|
||||
15 files in uploads/photos/
|
||||
|
||||
## Results
|
||||
|
||||
### Matched (1 photo → 1 asset update)
|
||||
|
||||
| Photo | Asset ID | Machine ID | Name | Action |
|
||||
|-------|----------|------------|------|--------|
|
||||
| keurig_label.jpg | 5144 | 636671 | 636671 / 956 Cypress Way | ✅ photo_path set to /uploads/photos/keurig_label.jpg |
|
||||
|
||||
### Unmatched — For Investigation (2 photos)
|
||||
|
||||
**coca_cola_label.jpg** — Coca-Cola branded label
|
||||
- Extracted IDs: RY10006338, RVCC-660-8, 201037BA00039
|
||||
- RY10006338 is Royal-pattern serial but not in DB
|
||||
- 201037BA00039 is Royal serial format; closest DB match: Asset #6815 (A201037BA00030)
|
||||
- Likely a pulled/retired or unregistered machine
|
||||
|
||||
**telemetry_device.jpg** — 4G telemetry device
|
||||
- Extracted IDs: 4434331624226353 (ICCID), 48602143 (Monyx ID)
|
||||
- These are telemetry/sim identifiers, not vending machine assets
|
||||
- Consider tracking in a separate telemetry devices table
|
||||
|
||||
### Test Data (3 photos)
|
||||
- 557ba8d1..., 59dd8174..., a8c7e553...: "Machine 1D 12845-678901" — app test pattern, no DB match
|
||||
|
||||
### No Text (9 photos)
|
||||
- Icon-sized images (100x100 to 200x100): no readable text
|
||||
|
||||
## Files Changed
|
||||
- `/home/oplabs/projects/canteen-asset-tracker/assets.db` — Asset #5144 photo_path updated
|
||||
- `/home/oplabs/projects/canteen-asset-tracker/scripts/crossref_photos.py` — reusable batch matching script
|
||||
+2
-2
@@ -22,7 +22,7 @@ Three ways to add an asset:
|
||||
| Method | Description |
|
||||
|--------|-------------|
|
||||
| **Barcode** | Scan barcodes via device camera (ZXing library). Found assets redirect to detail view; unknown codes prompt to create new. |
|
||||
| **OCR** | Upload a photo of a machine ID sticker; text is extracted via Tesseract OCR. |
|
||||
|| **OCR** | Upload a photo of a machine ID sticker; text is extracted via Tesseract OCR or Ollama vision model (qwen2.5vl:3b on Windows PC). When Tesseract fails (dark/complex labels), falls back to Ollama for much more accurate text extraction. |
|
||||
| **Manual** | Full form with all fields. |
|
||||
|
||||

|
||||
@@ -199,6 +199,6 @@ Returns all geofence service areas assigned to a specific user — useful for fi
|
||||
| **Maps** | Leaflet 1.9.4 + OpenStreetMap |
|
||||
| **Geofences** | Leaflet Draw + point-in-polygon (Turf.js) |
|
||||
| **Scanner** | ZXing (CDN) |
|
||||
| **OCR** | Tesseract via pytesseract |
|
||||
| **OCR** | Ollama qwen2.5vl:3b (primary) + Tesseract (fallback) — via SSH tunnel to Windows PC |
|
||||
| **TLS** | Self-signed cert on port 8901 |
|
||||
| **Reverse Proxy** | Nginx Proxy Manager Plus |
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
# Field Map — Canteen Asset Tracker
|
||||
|
||||
Every column in `assets.db`, how each pipeline touches it.
|
||||
|
||||
## Source Legend
|
||||
|
||||
| Source | Pipeline | File(s) | How to Run |
|
||||
|--------|----------|---------|------------|
|
||||
| **MSFS** | ADB pull → Dynamics 365 extraction | `pull_msfs_data.py` → `sync_assets_to_canteen.py` | Button on extraction viewer `:8910` **Manual** |
|
||||
| **CANT** | Cantaloupe staged import | `cantaloupe_sync.py` in admin server | Dashboard upload **Manual** |
|
||||
| **SEED** | Seed Excel import | `parser.py` → `db_writer.py` → `sync.py` | CLI `--mode=seed` or `--mode=update` **Manual** |
|
||||
| **APP** | Main app (technician check-in, photo, GPS) | `server.py` | Used by techs in field |
|
||||
| **ADMIN** | Admin dashboard edits | `admin_server.py` | Dashboard Cleanup/Mass Edit |
|
||||
|
||||
## Policy Legend
|
||||
|
||||
- **always** — pipeline overwrites this field every run
|
||||
- **if_empty** — only filled when currently NULL/empty
|
||||
- **never** — pipeline never touches this column
|
||||
- **if_numeric** — only overwritten if value is bare number (Cantaloupe default ID)
|
||||
|
||||
---
|
||||
|
||||
## 🔵 MSFS Sync — Field Policy
|
||||
|
||||
| Column | Policy | Dynamics 365 Source |
|
||||
|--------|--------|---------------------|
|
||||
| `serial_number` | always | `hsl_serialnumber` |
|
||||
| `connect_id` | always | `hsl_connectid` |
|
||||
| `equipment_id` | always | `hsl_equipmentid` |
|
||||
| `canteen_connect_guid` | always | `hsl_canteenconnectguid` |
|
||||
| `barcode` | always | `hsl_barcode` |
|
||||
| `manufacturer` | always | `hsl_manufacturertext` |
|
||||
| `model` | always | `hsl_modeltext` or `hsl_modelname` |
|
||||
| `route_name` | always | `hsl_routeidname` |
|
||||
| `install_date` | always | `hsl_opendate` |
|
||||
| `device` | always | `msdyn_deviceid` |
|
||||
| `name` | if_numeric | `msdyn_name` or `hsl_name` |
|
||||
| `company` | if_empty | Account `name` |
|
||||
| `customer_name` | if_empty | Account `name` |
|
||||
| `place` | if_empty | Account `address1_city` |
|
||||
| `address` | if_empty | Account `address1_line1` + `line2` |
|
||||
| `make` | **never** | _(uses `manufacturer` → separate column)_ |
|
||||
| `building_name` | **never** | — |
|
||||
| `floor` | **never** | — |
|
||||
| `room` | **never** | — |
|
||||
| `building_number` | **never** | — |
|
||||
| `trailer_number` | **never** | — |
|
||||
| `location_area` | **never** | — |
|
||||
| `latitude` | **never** | _(mapped but only if NULL in code)_ |
|
||||
| `longitude` | **never** | _(mapped but only if NULL in code)_ |
|
||||
| `photo_path` | **never** | — |
|
||||
| `category` | **never** | — |
|
||||
| `status` | **never** | — |
|
||||
| `description` | **never** | — |
|
||||
|
||||
> All other columns (sales data, alerts, disney flags, etc.) are **never** touched by MSFS sync.
|
||||
|
||||
---
|
||||
|
||||
## 🟡 Cantaloupe Sync — Field Policy
|
||||
|
||||
| Column | Policy | Notes |
|
||||
|--------|--------|-------|
|
||||
| `machine_id` | never (match key) | — |
|
||||
| `serial_number` | always | — |
|
||||
| `name` | always | — |
|
||||
| `description` | always | — |
|
||||
| `category` | always | Validated against categories table |
|
||||
| `status` | always | Validated against statuses |
|
||||
| `make` | always | — |
|
||||
| `model` | always | — |
|
||||
| `address` | always | — |
|
||||
| `building_name` | always | — |
|
||||
| `building_number` | always | — |
|
||||
| `floor` | always | — |
|
||||
| `room` | always | — |
|
||||
| `trailer_number` | always | — |
|
||||
| `walking_directions` | always | — |
|
||||
| `map_link` | always | — |
|
||||
| `parking_location` | always | — |
|
||||
| `photo_path` | always | — |
|
||||
| `latitude` | always | — |
|
||||
| `longitude` | always | — |
|
||||
| `geofence_radius_meters` | always | — |
|
||||
| `dex_report_date` | always | — |
|
||||
| `deployed` | always | — |
|
||||
| `pulled_date` | always | — |
|
||||
| `customer_id` | always | Via customer name lookup/creation |
|
||||
| `location_id` | always | Via location name lookup/creation |
|
||||
| `is_disney` | always | Derived from customer name prefix `D-` |
|
||||
|
||||
> **No preserve logic.** Any non-empty field in the import file overwrites the existing value.
|
||||
|
||||
---
|
||||
|
||||
## 🟢 Seed Import — Field Policy (db_writer.py)
|
||||
|
||||
### Seed Mode (`--mode=seed`)
|
||||
**DESTROYS AND RE-INSERTS** — `DELETE FROM assets` first, then inserts all rows fresh.
|
||||
|
||||
### Update Mode (`--mode=update`)
|
||||
Upsert by `machine_id`. **MSFS is primary data source** — seed fills gaps only.
|
||||
|
||||
**ALL fields** are treated as ONLY_IF_EMPTY for existing records:
|
||||
| Column | Policy | Excel Source / Generation |
|
||||
|--------|--------|--------------------------|
|
||||
| `serial_number` | if_empty | Serial Number column + OCR corrections |
|
||||
| `name` | if_empty | Generated: `{make} @ {place}` |
|
||||
| `make` | if_empty | Parsed from Type column |
|
||||
| `model` | if_empty | Parsed from Type column |
|
||||
| `class` | if_empty | Parsed from Type column |
|
||||
| `is_glass_front` | if_empty | Parsed from Type column |
|
||||
| `company` | if_empty | Parsed from Customer column |
|
||||
| `place` | if_empty | Parsed from Place column |
|
||||
| `building_name` | if_empty | Parsed from Place column |
|
||||
| `suite` | if_empty | Parsed from Place column |
|
||||
| `room` | if_empty | Parsed from Place column |
|
||||
| `trailer` | if_empty | Parsed from Place column |
|
||||
| `floor` | if_empty | Parsed from Place column |
|
||||
| `zone` | if_empty | Parsed from Place column |
|
||||
| `zone_type` | if_empty | Parsed from Place column |
|
||||
| `address` | if_empty | Address column |
|
||||
| `location_area` | if_empty | City column |
|
||||
| `latitude` | if_empty | Excel (existing GPS preserved) |
|
||||
| `longitude` | if_empty | Excel (existing GPS preserved) |
|
||||
| `photo_path` | if_empty | Excel (existing photo preserved) |
|
||||
| `install_date` | if_empty | Added Date (existing date preserved) |
|
||||
| `deployed` | if_empty | Deployed column (existing preserved) |
|
||||
| `pulled_date` | if_empty | Pulled Date (existing preserved) |
|
||||
| `dex_report_date` | if_empty | Last Dex Report Time |
|
||||
| `disney_park` | if_empty | Parsed from Customer column |
|
||||
| `disney_area` | if_empty | Derived from disney_park |
|
||||
| `remote_pricing_status` | if_empty | Status column |
|
||||
| `alerts` | if_empty | Alerts column + Coil/Product Alerts |
|
||||
| `telemetry_provider` | if_empty | Parsed from Device column |
|
||||
| `card_reader_brand` | if_empty | Parsed from Device column |
|
||||
| `priority` | if_empty | Computed from yearly_sales |
|
||||
| `yearly_sales` | if_empty | Yearly Sales column |
|
||||
| `monthly_sales` | if_empty | Monthly Sales column |
|
||||
| `weekly_sales` | if_empty | Weekly Sales column |
|
||||
| `days_since_restock` | if_empty | Days Since Restock column |
|
||||
| `prepick_group` | if_empty | Prepick Group column |
|
||||
| `has_cashless` | if_empty | Has Cashless column |
|
||||
|
||||
> **New assets** (not found in DB by machine_id) are inserted with full seed data.
|
||||
> **Existing assets**: only empty/null fields are filled. Any field already populated by MSFS, cleanup, or field use is preserved.
|
||||
|
||||
---
|
||||
|
||||
## 🟠 Seed Import — Sync (`sync.py` → main app DB)
|
||||
|
||||
After `db_writer.py`, this script pushes the seed DB into the main app's schema.
|
||||
**Now follows ONLY_IF_EMPTY policy** for existing assets — same as db_writer update mode.
|
||||
|
||||
| Column | Policy |
|
||||
|--------|--------|
|
||||
| `machine_id` | never (match key) |
|
||||
| `serial_number` | if_empty |
|
||||
| `name` | if_empty |
|
||||
| `description` | if_empty |
|
||||
| `category` | if_empty |
|
||||
| `status` | if_empty |
|
||||
| `make` | if_empty |
|
||||
| `model` | if_empty |
|
||||
| `address` | if_empty |
|
||||
| `building_name` | if_empty |
|
||||
| `building_number` | if_empty |
|
||||
| `floor` | if_empty |
|
||||
| `room` | if_empty |
|
||||
| `trailer_number` | if_empty |
|
||||
| `walking_directions` | if_empty |
|
||||
| `map_link` | if_empty |
|
||||
| `parking_location` | if_empty |
|
||||
| `photo_path` | if_empty |
|
||||
| `latitude` | if_empty |
|
||||
| `longitude` | if_empty |
|
||||
| `geofence_radius_meters` | if_empty |
|
||||
| `disney_park` | if_empty |
|
||||
| `is_disney` | if_empty |
|
||||
| `dex_report_date` | if_empty |
|
||||
| `install_date` | if_empty |
|
||||
| `deployed` | if_empty |
|
||||
| `pulled_date` | if_empty |
|
||||
| `customer_id` | if_empty (via upsert customers table) |
|
||||
| `location_id` | if_empty (via upsert locations table) |
|
||||
|
||||
> **Also creates/updates** `customers` and `locations` tables.
|
||||
|
||||
---
|
||||
|
||||
## ⚪ Admin Tools — Policy
|
||||
|
||||
### Batch Update (Cleanup page)
|
||||
- `make`, `model`, `name`, `company`, `place`, `building_name`, `building_number`, `floor`, `room`, `location_area`, `address`, `category`, `status`, `customer_name`
|
||||
- **Always overwrites** with user-provided value.
|
||||
|
||||
### Clear GPS
|
||||
- `latitude`, `longitude` — set to NULL
|
||||
|
||||
### Mass Edit (existing tool)
|
||||
- Same allowed fields as Batch Update.
|
||||
|
||||
---
|
||||
|
||||
## ⚫ Main App (Tech Field) — What Touches What
|
||||
|
||||
| Action | Fields Affected |
|
||||
|--------|----------------|
|
||||
| Barcode scan check-in | `updated_at` |
|
||||
| GPS check-in | `latitude`, `longitude`, `updated_at` |
|
||||
| Photo upload | `photo_path`, `updated_at` |
|
||||
| OCR scan (name search) | (read-only) |
|
||||
| Register new asset | All columns with defaults |
|
||||
|
||||
---
|
||||
|
||||
## Pipeline Activity Status
|
||||
|
||||
| Pipeline | Currently Scheduled? | Frequency | Risk to Cleanup |
|
||||
|----------|--------------------|-----------|-----------------|
|
||||
| **MSFS Sync** | 🔴 **Manual only** | On-demand via extraction viewer | ✅ GPS never touched. Preserves name/company/place/building/floor. Only `model` overwritten. |
|
||||
| **Cantaloupe Sync** | 🔴 **No cron** | Manual upload on admin dashboard | ⚠️ Preserves GPS now (fixed — none from Cantaloupe). Still overwrites other fields. |
|
||||
| **Seed Import** `--mode=update` | 🔴 **Manual CLI only** | `python3 main.py -m update` | ✅ **GPS never touched** (check-in only). Fills gaps only for other fields. |
|
||||
| **Seed Import** `--mode=seed` | 🔴 **Manual CLI only** | `python3 main.py -m seed` | 🔥 **NUCLEAR** — `DELETE FROM assets`. Full re-insert. GPS set to NULL. |
|
||||
@@ -0,0 +1,174 @@
|
||||
# OCR / Vision Pipeline
|
||||
|
||||
> End-to-end: camera capture → text extraction → asset matching → auto-update.
|
||||
|
||||
The Canteen Asset Tracker has a multi-stage OCR pipeline that reads machine ID sticker photos, cross-references extracted identifiers against the assets database, and automatically updates matched assets with photo paths and serial numbers.
|
||||
|
||||
## Pipeline Stages
|
||||
|
||||
```
|
||||
Camera/Gallery Photo
|
||||
│
|
||||
▼
|
||||
┌──────────────────┐ Stage 1: Vision (Ollama qwen2.5vl:3b)
|
||||
│ Ollama Vision │ Runs first — most accurate on complex labels
|
||||
│ (Python API) │ Runs on Windows PC (RTX 2080), tunneled via SSH
|
||||
└────────┬─────────┘ to localhost:11434
|
||||
│ (fallback if text < 10 clean chars)
|
||||
▼
|
||||
┌──────────────────┐ Stage 2: OCR (Tesseract)
|
||||
│ Tesseract OCR │ Fallback — works on clean, high-contrast labels
|
||||
│ (pytesseract) │ Uses `--psm 6` (uniform block of text)
|
||||
└────────┬─────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────┐ Stage 3: Identifier Extraction
|
||||
│ Normalize & │ - Strips label prefixes (S/N:, ID#, Machine ID, etc.)
|
||||
│ Extract IDs │ - Removes punctuation, uppercases
|
||||
└────────┬─────────┘ - Extracts full lines + individual tokens
|
||||
│
|
||||
▼
|
||||
┌──────────────────┐ Stage 4: DB Cross-Reference
|
||||
│ Match against │ - Matches normalized IDs against serial_number,
|
||||
│ assets.db │ machine_id, connect_id, equipment_id, barcode
|
||||
└────────┬─────────┘ - Deduplicates by asset id
|
||||
│
|
||||
▼
|
||||
┌──────────────────┐ Stage 5: Auto-Update
|
||||
│ Write Back to │ - photo_path → set on matched assets (if blank)
|
||||
│ Matched Assets │ - serial_number → extracted from OCR text (if blank)
|
||||
└──────────────────┘ - GPS coords → set on matched asset (if blank, from EXIF)
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### POST `/api/ocr` — Full OCR pipeline
|
||||
|
||||
Accepts an image upload (multipart/form-data). Returns extracted text, matched assets, and auto-updates the database.
|
||||
|
||||
**Request:**
|
||||
```
|
||||
POST /api/ocr
|
||||
Content-Type: multipart/form-data
|
||||
file: <image> # Required: the sticker photo
|
||||
exif_data: <JSON string> # Optional: client-side EXIF data for gallery uploads
|
||||
```
|
||||
|
||||
**Response fields:**
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `raw_text` | Raw text extracted by Ollama or Tesseract |
|
||||
| `ocr_source` | `"ollama"`, `"tesseract"`, or `"none"` |
|
||||
| `machine_id` | 5-digit machine ID from Connect-ID pattern (XXXXX-XXXXXX) |
|
||||
| `confidence` | `"high"` (exact pattern match), `"low"` (loose match), `"none"` |
|
||||
| `matched_assets` | Array of assets matched via identifier cross-reference |
|
||||
| `exif_gps` | GPS coordinates from photo EXIF (if present) |
|
||||
| `gps_saved` | `true` if GPS was auto-saved to the matched asset |
|
||||
| `path` | Saved photo URL path (when `exif_data` provided) |
|
||||
| `photo_saved` | Number of matched assets whose `photo_path` was auto-updated |
|
||||
| `serial_saved` | Serial number value that was auto-saved to blank-serial matched assets |
|
||||
|
||||
### POST `/api/upload/photo` — Photo-only upload
|
||||
|
||||
Saves a photo without OCR. Returns the saved path and any EXIF GPS data.
|
||||
Use this when the photo was already matched to an asset client-side.
|
||||
|
||||
### POST `/api/match-text` — Text matching only
|
||||
|
||||
Accepts raw text and finds matching assets. Does not run OCR.
|
||||
Use for barcode scanner input, QR codes, or client-side vision results.
|
||||
|
||||
## Auto-Update Logic
|
||||
|
||||
When a photo is uploaded through `/api/ocr` and matches database assets:
|
||||
|
||||
1. **photo_path** — If the matched asset has no `photo_path` set, the saved photo's URL path is written to the asset. This links the photo directly to the asset record for display in the asset detail view.
|
||||
|
||||
2. **serial_number** — If the matched asset has a blank `serial_number` and the OCR text contains a serial-number pattern (prefixed with S/N, Serial#, Equipment ID, etc.), the extracted serial is written to all matched blank-serial assets.
|
||||
|
||||
3. **GPS coordinates** — If the photo has EXIF GPS data and the matched asset has no lat/lng coordinates, they are auto-saved.
|
||||
|
||||
All auto-updates are best-effort and non-critical — they don't fail the OCR endpoint if a DB write errors.
|
||||
|
||||
## Backfill Script
|
||||
|
||||
### `scripts/backfill_vision_photos.py`
|
||||
|
||||
Processes all photos in `uploads/photos/` that aren't already linked to an asset via `photo_path`. Useful for:
|
||||
|
||||
- Processing photos that were uploaded before the auto-link feature was added
|
||||
- Retrying photos that failed to match initially
|
||||
- Bulk-linking a directory of asset photos
|
||||
|
||||
```bash
|
||||
# Dry-run: see what would be updated
|
||||
python3 scripts/backfill_vision_photos.py
|
||||
|
||||
# Apply changes
|
||||
python3 scripts/backfill_vision_photos.py --apply
|
||||
|
||||
# Force overwrite existing photo_path values
|
||||
python3 scripts/backfill_vision_photos.py --apply --force
|
||||
```
|
||||
|
||||
Options:
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--apply` | Write changes to DB (default is dry-run) |
|
||||
| `--force` | Overwrite existing `photo_path` values on assets |
|
||||
| `--db` | Custom database path |
|
||||
| `--photos-dir` | Custom photos directory |
|
||||
|
||||
## `scripts/match_label_photo.py`
|
||||
|
||||
CLI utility to OCR a single image and display matched assets. Supports `--text` flag to skip OCR and pass raw text directly.
|
||||
|
||||
```bash
|
||||
# OCR a photo
|
||||
python3 scripts/match_label_photo.py path/to/photo.jpg
|
||||
|
||||
# Match against pre-extracted text
|
||||
python3 scripts/match_label_photo.py --text "S/N: 2500.0100.0025534"
|
||||
```
|
||||
|
||||
## OCR Services
|
||||
|
||||
### Primary: Ollama Vision (qwen2.5vl:3b)
|
||||
|
||||
- **Host:** Windows gaming PC (RTX 2080), tunneled to `localhost:11434`
|
||||
- **Accuracy:** High — can read text from complex labels, dark backgrounds, angled photos
|
||||
- **Speed:** ~5-15 seconds per image (depends on GPU load)
|
||||
- **Endpoint:** `/api/generate` with prompt focused on text extraction
|
||||
|
||||
### Fallback: Tesseract OCR
|
||||
|
||||
- **Installation:** `sudo apt-get install -y tesseract-ocr`
|
||||
- **Python:** `pip install pytesseract Pillow`
|
||||
- **Accuracy:** Good on clean, high-contrast, well-lit labels
|
||||
- **Mode:** `--psm 6` (assume uniform block of text)
|
||||
|
||||
### Availability Check
|
||||
|
||||
The server checks both services at startup:
|
||||
- `_HAS_OLLAMA` — set if `http://127.0.0.1:11434/api/generate` responds
|
||||
- `_HAS_TESSERACT` — set if `pytesseract` is importable
|
||||
|
||||
## Identifier Matching
|
||||
|
||||
The `normalize_identifier()` function in `classify_makes.py` handles identifier normalization:
|
||||
|
||||
1. Strips label prefixes (`S/N:`, `ID#`, `Machine ID`, `Monyx ID`, etc.)
|
||||
2. Removes dots, dashes, spaces, slashes, colons
|
||||
3. Uppercases everything
|
||||
4. Returns just the alphanumeric core
|
||||
|
||||
This normalized string is then searched across `serial_number`, `connect_id`, `equipment_id`, `machine_id`, and `barcode` columns in the assets table.
|
||||
|
||||
## Photo Storage
|
||||
|
||||
- **Directory:** `uploads/photos/`
|
||||
- **Naming:** UUID hex (avoids filename collisions)
|
||||
- **Extensions:** `.png`, `.jpg`, `.jpeg`, `.dng` (configurable via `PHOTO_ALLOWED_EXTS`)
|
||||
- **Max size:** 20 MB (configurable via `PHOTO_MAX_SIZE`)
|
||||
- **Serving:** Mounted at `/uploads` via FastAPI StaticFiles
|
||||
- **EXIF round-trip:** Client reads EXIF before upload, server re-embeds it into saved JPEG
|
||||
@@ -0,0 +1,327 @@
|
||||
# MSFS Data Import — End-to-End Pipeline
|
||||
|
||||
> How Microsoft Dynamics 365 Field Service data was extracted, merged with Cantaloupe data, and imported into the Canteen Asset Tracker.
|
||||
>
|
||||
> **Date:** 2026-05-28 (initial), 2026-05-28 (clean-slate reset)
|
||||
> **Assets in DB:** 9,636
|
||||
> **Assets with real GPS:** 28 (field-collected, not address-geocoded)
|
||||
> **GPS backup:** `/tmp/gps_restore.sql` (28 UPDATE statements by machine_id)
|
||||
> **Pre-import backup:** `assets.db.20260528_225934.pre-msfs-import`
|
||||
> **Note:** All GPS from MSFS (address-geocoded) was cleared during the clean-slate reset. Only real field GPS (from technician photo EXIF) remains.
|
||||
|
||||
---
|
||||
|
||||
## 1. Extraction from Dynamics 365
|
||||
|
||||
The Dynamics 365 Field Service mobile app (Android) on a rooted Pixel 4a was the data source. The offline SQLite databases stored on the device contain full schema snapshots — `msdyn_customerasset`, `msdyn_workorder`, `account`, `bookableresourcebooking`, and 167+ other tables.
|
||||
|
||||
### 1.1 Phone → Disk
|
||||
|
||||
```bash
|
||||
# Pull the main offline DB from the device
|
||||
adb shell "su -c cp /data/data/com.microsoft.crm.crmphone.fieldServices/databases/msfs_backup.db /sdcard/"
|
||||
adb pull /sdcard/msfs_backup.db data-samples/
|
||||
|
||||
# 1.7 GB SQLite database, pulled 2026-05-28
|
||||
```
|
||||
|
||||
### 1.2 Dynamics 365 API Extraction (Live)
|
||||
|
||||
A Service Principal application was registered in Entra ID (same tenant as Field Service) with client credentials flow. The `dynamics_token.py` manager auto-refreshes tokens stored in `pass`.
|
||||
|
||||
```bash
|
||||
# Get a token and pull equipment, accounts, work orders via Dataverse WebAPI
|
||||
python3 dynamics_token.py # Outputs Bearer token
|
||||
python3 query_booking_gps.py # Booking GPS from live API
|
||||
```
|
||||
|
||||
**Results from live API:**
|
||||
| Entity | Records |
|
||||
|---|---|
|
||||
| Equipment (`msdyn_customerasset`) | 14,440 |
|
||||
| Accounts (`account`) | 5,305 |
|
||||
| Work Orders (`msdyn_workorder`) | 15,531 |
|
||||
|
||||
### 1.3 Photo Extraction
|
||||
|
||||
3,703 technician photos were extracted from the offline DB's `annotation` / `activitymimeattachment` binary blobs to disk at `web/static/photos/`. These are actual field photos taken by technicians on their phones, containing:
|
||||
|
||||
- **EXIF GPS** from the phone camera (176 photos)
|
||||
- **ConnectID stickers** and barcode images
|
||||
- Work order photos of machines at customer sites
|
||||
|
||||
---
|
||||
|
||||
## 2. Merge (MSFS + Canteen)
|
||||
|
||||
### 2.1 The Merge Script
|
||||
|
||||
`standalone-merge` joins MSFS and Canteen records by **serial number** (`hsl_serialnumber`). No other common key exists.
|
||||
|
||||
**Source tables from MSFS:**
|
||||
- `msdyn_customerasset` — equipment records with serials, GPS, manufacturer, model, ConnectID
|
||||
- `msdyn_workorder` — work order history linked to assets
|
||||
- `account` — customer account details with geocoded addresses
|
||||
|
||||
**Source from Canteen:**
|
||||
- Old `assets.db` (1,848 records from Cantaloupe CSV import)
|
||||
|
||||
### 2.2 Join Logic
|
||||
|
||||
```python
|
||||
# standalone-merge → merge()
|
||||
all_serials = set(msfs_assets) | set(canteen_assets)
|
||||
|
||||
for serial in sorted(all_serials):
|
||||
if msfs and canteen: match = "joined" # Both sources have this asset
|
||||
elif msfs: match = "msfs_only" # Only MSFS knows about it
|
||||
else: match = "canteen_only" # Only Cantaloupe import has it
|
||||
```
|
||||
|
||||
Fields are prefixed by source: `msfs_*` for MS Field Service fields, `canteen_*` for Cantaloupe import fields. Work orders are attached per-asset.
|
||||
|
||||
### 2.3 Merge Results
|
||||
|
||||
| Metric | Count |
|
||||
|---|---|
|
||||
| **Total merged records** | 10,038 |
|
||||
| **Joined** (both sources, matched by serial) | 1,832 |
|
||||
| **MSFS-only** (no Cantaloupe match) | 8,204 |
|
||||
| **Canteen-only** (no MSFS match) | 2 |
|
||||
| **Work orders linked** to assets | 9,652 |
|
||||
| **Assets with work orders** | 3,199 |
|
||||
| **Accounts resolved** | 3,486 |
|
||||
| **Customers resolved** | 284 |
|
||||
| **Locations resolved** | 364 |
|
||||
|
||||
**Timing:** The "MSFS-only" dominance means the old Cantaloupe import had very limited coverage. Most of the 8,204 MSFS-only assets are from Dynamics-only equipment not managed in the Cantaloupe system.
|
||||
|
||||
---
|
||||
|
||||
## 3. Field Mapping
|
||||
|
||||
### 3.1 GPS Resolution (Priority Order)
|
||||
|
||||
When importing into the assets table, GPS coordinates are resolved in this order:
|
||||
|
||||
1. **MSFS latitude/longitude** from `msdyn_customerasset` (if present)
|
||||
2. **Canteen latitude/longitude** from old imports (if MSFS missing)
|
||||
3. If both missing → `NULL` (no GPS — 9,608 assets fall here)
|
||||
|
||||
The GPS in `msdyn_customerasset` itself is **geocoded account addresses**, not real device GPS. Real EXIF GPS from technician photos would need the OCR pipeline (see §4).
|
||||
|
||||
### 3.2 Field Mapping Spec
|
||||
|
||||
| `assets` column | Source | MSFS field |
|
||||
|---|---|---|
|
||||
| `machine_id` | msfs_machine_id → canteen_machine_id | Extracted from `hsl_equipmentid` suffix |
|
||||
| `serial_number` | serial_number | `hsl_serialnumber` |
|
||||
| `name` | msfs_name → canteen_name | `msdyn_name` |
|
||||
| `make` | msfs_manufacturer | `hsl_manufacturertext` |
|
||||
| `model` | msfs_dex_model | `hsl_dexmodel` |
|
||||
| `latitude` / `longitude` | msfs → canteen | `msdyn_latitude` / `msdyn_longitude` |
|
||||
| `address` | account.address | From `account` table |
|
||||
| `connect_id` | msfs_connect_id | `hsl_connectid` |
|
||||
| `canteen_connect_guid` | msfs_canteen_connect_guid | `hsl_canteenconnectguid` |
|
||||
| `manufacturer` | msfs_manufacturer | `hsl_manufacturertext` |
|
||||
| `equipment_id` | msfs_equipment_id | `hsl_equipmentid` |
|
||||
| `company` | canteen_company | Old Cantaloupe import |
|
||||
| `category` | canteen_category | Old Cantaloupe import |
|
||||
| `install_date` | canteen → msfs | `hsl_opendate` |
|
||||
|
||||
### 3.3 Output JSON
|
||||
|
||||
`web/static/data/merged-assets.json` — 10,038 records (159 MB) with full field mapping, work order attachments, account/customer/location resolution, and GPS coordinates.
|
||||
|
||||
---
|
||||
|
||||
## 4. OCR & Photo EXIF GPS Pipeline
|
||||
|
||||
### 4.1 Problem
|
||||
|
||||
- 9,608 of 9,636 assets have no real GPS
|
||||
- Existing GPS is geocoded addresses (account-based, not actual machine location)
|
||||
- 176 photos contain real EXIF GPS from technician phone cameras
|
||||
|
||||
### 4.2 Approach
|
||||
|
||||
The plan in `docs/plans/2026-05-28-photo-exif-gps-pipeline.md` outlines a multi-pass OCR pipeline:
|
||||
|
||||
1. **Tesseract OCR** on all 3,703 photos — 4-tier cascade (scales, preprocessing modes, PSM variants)
|
||||
2. **Vision API fallback** (`mimo-v2-omni`) for Tesseract failures
|
||||
3. **Machine ID extraction** from ConnectID stickers (`XXXXX-YYYYYY`), serial plates, barcodes
|
||||
4. **Cross-reference** OCR results with photo EXIF GPS
|
||||
5. **Write high-confidence** updates to `assets.db` with `gps_source = 'photo_exif'`
|
||||
|
||||
**Current status:** 201 photos OCR'd as proof of concept (5.4%), 14 unique machine IDs identified.
|
||||
|
||||
### 4.3 Key Scripts
|
||||
|
||||
| Script | Purpose |
|
||||
|---|---|
|
||||
| `ocr_local.py` | Tesseract 4-tier local OCR cascade |
|
||||
| `ocr_batch.py` | Vision API batch OCR (OpenCode Go) |
|
||||
| `ocr_mapping.py` | OCR results → machine ID → canteen asset mapping |
|
||||
| `consolidate_gps.py` | Multi-source GPS consolidation |
|
||||
| `_ocr_analyze.py` | OCR failure analysis |
|
||||
| `standalone-merge` | MSFS → Canteen merge |
|
||||
|
||||
---
|
||||
|
||||
## 5. Import Script
|
||||
|
||||
### `import_msfs.py`
|
||||
|
||||
The stand-alone import script located at `~/projects/canteen-asset-tracker/import_msfs.py` performs:
|
||||
|
||||
1. **Backup** existing `assets.db` → `assets.db.{timestamp}.pre-msfs-import`
|
||||
2. **Read** `merged-assets.json` (10,038 records)
|
||||
3. **Create fresh** `assets.db` with full v2 schema (18 tables, indexes, triggers)
|
||||
4. **Map** merged fields to assets table columns per the mapping spec above
|
||||
5. **Deduplicate** by `machine_id` with priority: joined > msfs_only > canteen_only
|
||||
6. **Insert** categories, customers, locations from merged data
|
||||
7. **Insert** default users (admin/technician)
|
||||
8. **Verify** with table/row/index counts
|
||||
|
||||
### Run
|
||||
|
||||
```bash
|
||||
cd ~/projects/canteen-asset-tracker
|
||||
python3 import_msfs.py
|
||||
```
|
||||
|
||||
### DB Stats After Import
|
||||
|
||||
```sql
|
||||
Assets total: 9,636
|
||||
Assets with GPS: 28
|
||||
Categories: 21
|
||||
Customers: 284
|
||||
Locations: 364
|
||||
Users: 2 (admin, tech)
|
||||
Table count: 18
|
||||
Index count: 6
|
||||
DB size: ~11 MB (WAL mode)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Extraction DB (Live Route Planning)
|
||||
|
||||
The extraction DB is linked from the canteen asset tracker server for live work order and route optimization queries:
|
||||
|
||||
```python
|
||||
# server.py → _get_extraction_db()
|
||||
EXTRACTION_DB = (
|
||||
Path.home()
|
||||
/ "projects/ms-field-service-extraction"
|
||||
/ "data-samples/msfs_backup"
|
||||
/ "fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db"
|
||||
)
|
||||
```
|
||||
|
||||
Five endpoints use this:
|
||||
| Endpoint | Purpose |
|
||||
|---|---|
|
||||
| `GET /api/workorders/search` | Paginated search by name/account/city |
|
||||
| `POST /api/workorders/lookup` | Bulk lookup by work order ID/name |
|
||||
| `GET /api/workorders/today` | Today's active bookings by technician |
|
||||
| `GET /api/workorders/technicians` | Distinct technician list |
|
||||
| `POST /api/route/optimize` | TSP route optimization (nearest-neighbor + 2-opt) |
|
||||
|
||||
---
|
||||
|
||||
## 7. File Locations
|
||||
|
||||
| Artifact | Path |
|
||||
|---|---|
|
||||
| MSFS backup DB | `~/projects/ms-field-service-extraction/data-samples/msfs_backup/` |
|
||||
| Merged JSON | `~/projects/ms-field-service-extraction/web/static/data/merged-assets.json` |
|
||||
| Live API data | `data-samples/dynamics_equipment.json` (14,440), `dynamics_accounts.json` (5,305), `dynamics_workorders.json` (15,531) |
|
||||
| Photos | `~/projects/ms-field-service-extraction/web/static/photos/` (3,703 files) |
|
||||
| OCR results | `web/static/data/ocr_results.json` |
|
||||
| Merge script | `~/projects/ms-field-service-extraction/standalone-merge` |
|
||||
| Import script | `~/projects/canteen-asset-tracker/import_msfs.py` |
|
||||
| Canteen SQLite DB | `~/projects/canteen-asset-tracker/assets.db` |
|
||||
| Pre-import backup | `assets.db.20260528_214316.pre-msfs-import` |
|
||||
| GPS consolidation | `~/projects/ms-field-service-extraction/consolidate_gps.py` |
|
||||
| GPS consolidated JSON | `web/static/data/gps_consolidated_update.json` |
|
||||
| Data catalog | `~/projects/ms-field-service-extraction/DATA_CATALOG.md` |
|
||||
| Photo EXIF GPS plan | `docs/plans/2026-05-28-photo-exif-gps-pipeline.md` |
|
||||
|
||||
---
|
||||
|
||||
## 8. Architecture Diagram
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ Rooted Pixel 4a │
|
||||
│ (Shawn Canada's device) │
|
||||
│ MS Field Service Mobile App │
|
||||
│ Offline SQLite (1.7 GB, 167+ tbls) │
|
||||
└──────────────┬──────────────────────┘
|
||||
│ adb pull
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ MSFS Backup DB │
|
||||
│ data-samples/msfs_backup/ │
|
||||
│ - msdyn_customerasset (14K rows) │
|
||||
│ - msdyn_workorder (15K rows) │
|
||||
│ - account (5K rows) │
|
||||
│ - annotation (photos, 3.7K blobs) │
|
||||
└──────────┬──────────────────────────┘
|
||||
│
|
||||
┌────────────────┼────────────────────┐
|
||||
▼ ▼ ▼
|
||||
┌─────────────────┐ ┌─────────────┐ ┌─────────────────┐
|
||||
│ photo extraction │ │ merge │ │ Dynamics API │
|
||||
│ 3,703 JPEGs │ │ standalone │ │ (Service Prin.) │
|
||||
│ EXIF GPS → 176 │ │ -merge │ │ 14K equipment │
|
||||
│ → OCR pipeline │ │ serial join │ │ 15K work orders │
|
||||
└────────┬─────────┘ └──────┬──────┘ └─────────────────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ merged-assets.json │
|
||||
│ 10,038 records │
|
||||
│ 1,832 joined / 8,204 MSFS-only │
|
||||
│ 9,652 work orders linked │
|
||||
└──────────────────┬──────────────────────────┘
|
||||
│ import_msfs.py
|
||||
▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ assets.db │
|
||||
│ 9,636 assets (was 1,848) │
|
||||
│ 28 with GPS │
|
||||
│ 18 tables │
|
||||
│ 6 indexes │
|
||||
└──────────────────┬──────────────────────────┘
|
||||
│ server.py links for
|
||||
│ live route/work order
|
||||
▼
|
||||
┌─────────────────────────────────────────────┐
|
||||
│ Canteen Asset Tracker App │
|
||||
│ → Find Asset (barcode/OCR lookup) │
|
||||
│ → Map (Leaflet, GPS pins) │
|
||||
│ → Route (TSP optimization, extraction DB) │
|
||||
│ → Nav (OSRM driving/walking) │
|
||||
└─────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Future Work
|
||||
|
||||
The **Photo EXIF GPS Pipeline** (see `docs/plans/2026-05-28-photo-exif-gps-pipeline.md`) is designed to add real GPS to the remaining 9,608 assets:
|
||||
|
||||
| Task | Description | Status |
|
||||
|---|---|---|
|
||||
| 0 | Diagnose OCR bottleneck (why only 14/3,703 photos yield IDs) | ⬜ |
|
||||
| 1 | Fix OCR regex for ConnectID, serial, barcode formats | ⬜ |
|
||||
| 2 | Batch OCR all 3,703 photos with Tesseract (all photos, not just GPS) | ⬜ |
|
||||
| 3 | Vision API OCR on Tesseract failures | ⬜ |
|
||||
| 4 | Multi-source disambiguation (timestamps + location) | ⬜ |
|
||||
| 5 | Push verified GPS into `assets.db` with source tracking | ⬜ |
|
||||
| 6 | Verify and visualize on map | ⬜ |
|
||||
|
||||
Each photo with EXIF GPS that can be matched to a machine ID gives us a **real device GPS coordinate** — far more accurate than the current geocoded addresses.
|
||||
+630
@@ -0,0 +1,630 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import MSFS merged data into a fresh assets.db for the Canteen Asset Tracker.
|
||||
|
||||
Steps:
|
||||
1. Back up the existing assets.db with a timestamp suffix
|
||||
2. Read merged-assets.json (10,038 records from MSFS + Canteen merge)
|
||||
3. Create a fresh assets.db with the full schema (tables, indexes, triggers)
|
||||
4. Map merged fields to the assets table per the mapping specification
|
||||
5. Insert default users (admin + tech)
|
||||
6. Verify the data
|
||||
|
||||
Usage:
|
||||
python3 import_msfs.py
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────────
|
||||
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
DB_PATH = os.path.join(PROJECT_DIR, "assets.db")
|
||||
MERGED_JSON = os.path.expanduser(
|
||||
"~/projects/ms-field-service-extraction/web/static/data/merged-assets.json"
|
||||
)
|
||||
|
||||
# ── Password hashing (mirrors server.py) ──────────────────────────────────
|
||||
def hash_password(password: str) -> str:
|
||||
return hashlib.sha256(password.encode()).hexdigest()
|
||||
|
||||
# ── 1. Backup existing DB ────────────────────────────────────────────────
|
||||
def backup_db():
|
||||
if not os.path.exists(DB_PATH):
|
||||
print("[BACKUP] No existing assets.db found — skipping backup.")
|
||||
return None
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = f"{DB_PATH}.{timestamp}.pre-msfs-import"
|
||||
shutil.copy2(DB_PATH, backup_path)
|
||||
size_mb = os.path.getsize(backup_path) / (1024 * 1024)
|
||||
print(f"[BACKUP] Copied assets.db → {backup_path} ({size_mb:.1f} MB)")
|
||||
return backup_path
|
||||
|
||||
# ── 2. Read merged data ──────────────────────────────────────────────────
|
||||
def read_merged_data():
|
||||
if not os.path.exists(MERGED_JSON):
|
||||
print(f"[ERROR] Merged data not found at: {MERGED_JSON}")
|
||||
sys.exit(1)
|
||||
with open(MERGED_JSON, "r") as f:
|
||||
data = json.load(f)
|
||||
assets = data.get("assets", [])
|
||||
meta = data.get("meta", {})
|
||||
print(f"[READ] Loaded {len(assets)} records from merged-assets.json")
|
||||
print(f" Joined: {meta.get('joined', '?')} "
|
||||
f"MSFS-only: {meta.get('msfs_only', '?')} "
|
||||
f"Canteen-only: {meta.get('canteen_only', '?')}")
|
||||
return assets
|
||||
|
||||
# ── 3. Create fresh DB with full schema ──────────────────────────────────
|
||||
SCHEMA_SQL = """
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT UNIQUE NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
role TEXT NOT NULL DEFAULT 'technician',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS customers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS customer_contacts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
customer_id INTEGER NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
name TEXT,
|
||||
phone TEXT,
|
||||
email TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS locations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
customer_id INTEGER REFERENCES customers(id),
|
||||
name TEXT,
|
||||
address TEXT DEFAULT '',
|
||||
building_name TEXT DEFAULT '',
|
||||
building_number TEXT DEFAULT '',
|
||||
floor TEXT DEFAULT '',
|
||||
trailer_number TEXT DEFAULT '',
|
||||
site_hours TEXT DEFAULT '',
|
||||
access_notes TEXT DEFAULT '',
|
||||
walking_directions TEXT DEFAULT '',
|
||||
map_link TEXT DEFAULT '',
|
||||
latitude REAL DEFAULT NULL,
|
||||
longitude REAL DEFAULT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
disney_park TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rooms (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
location_id INTEGER NOT NULL REFERENCES locations(id) ON DELETE CASCADE,
|
||||
name TEXT,
|
||||
floor TEXT DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
icon TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS key_names (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS key_types (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS badge_types (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS makes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS models (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
make_id INTEGER NOT NULL REFERENCES makes(id),
|
||||
name TEXT NOT NULL,
|
||||
icon_path TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS assets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
machine_id TEXT NOT NULL UNIQUE,
|
||||
serial_number TEXT DEFAULT '',
|
||||
name TEXT NOT NULL,
|
||||
description TEXT DEFAULT '',
|
||||
category TEXT NOT NULL DEFAULT 'Other',
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
make TEXT DEFAULT '',
|
||||
model TEXT DEFAULT '',
|
||||
address TEXT DEFAULT '',
|
||||
building_name TEXT DEFAULT '',
|
||||
building_number TEXT DEFAULT '',
|
||||
floor TEXT DEFAULT '',
|
||||
room TEXT DEFAULT '',
|
||||
trailer_number TEXT DEFAULT '',
|
||||
walking_directions TEXT DEFAULT '',
|
||||
map_link TEXT DEFAULT '',
|
||||
parking_location TEXT DEFAULT '',
|
||||
photo_path TEXT,
|
||||
customer_id INTEGER REFERENCES customers(id),
|
||||
location_id INTEGER REFERENCES locations(id),
|
||||
assigned_to INTEGER REFERENCES users(id),
|
||||
latitude REAL DEFAULT NULL,
|
||||
longitude REAL DEFAULT NULL,
|
||||
geofence_radius_meters INTEGER DEFAULT 50,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
disney_park TEXT DEFAULT NULL,
|
||||
is_disney INTEGER DEFAULT 0,
|
||||
dex_report_date TEXT DEFAULT NULL,
|
||||
install_date TEXT DEFAULT NULL,
|
||||
deployed TEXT DEFAULT NULL,
|
||||
pulled_date TEXT DEFAULT NULL,
|
||||
company TEXT DEFAULT '',
|
||||
location_area TEXT DEFAULT '',
|
||||
place TEXT DEFAULT '',
|
||||
connect_id TEXT DEFAULT '',
|
||||
canteen_connect_guid TEXT DEFAULT '',
|
||||
manufacturer TEXT DEFAULT '',
|
||||
equipment_id TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS asset_keys (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asset_id INTEGER NOT NULL REFERENCES assets(id) ON DELETE CASCADE,
|
||||
key_name TEXT,
|
||||
key_type TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS asset_badges (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asset_id INTEGER NOT NULL REFERENCES assets(id) ON DELETE CASCADE,
|
||||
badge_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
key TEXT UNIQUE NOT NULL,
|
||||
value TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS activity_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
action TEXT,
|
||||
entity_type TEXT,
|
||||
entity_id INTEGER,
|
||||
details TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS geofences (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT,
|
||||
points TEXT,
|
||||
color TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS geofence_users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
geofence_id INTEGER NOT NULL REFERENCES geofences(id) ON DELETE CASCADE,
|
||||
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
UNIQUE(geofence_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS visits (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
asset_id INTEGER REFERENCES assets(id) ON DELETE CASCADE,
|
||||
checkin_time TEXT,
|
||||
checkout_time TEXT,
|
||||
duration_minutes INTEGER,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id INTEGER REFERENCES users(id) ON DELETE CASCADE,
|
||||
token TEXT UNIQUE NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
expires_at TEXT NOT NULL DEFAULT (datetime('now', '+1 day'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cantaloupe_sync_batches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
approved_at TEXT,
|
||||
file_path TEXT,
|
||||
row_count INTEGER DEFAULT 0,
|
||||
diff_summary TEXT,
|
||||
raw_data TEXT,
|
||||
error_message TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS service_entrances (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
location_id INTEGER NOT NULL REFERENCES locations(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
latitude REAL NOT NULL,
|
||||
longitude REAL NOT NULL,
|
||||
notes TEXT DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS checkins (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asset_id INTEGER REFERENCES assets(id) ON DELETE CASCADE,
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
latitude REAL,
|
||||
longitude REAL,
|
||||
accuracy REAL,
|
||||
photo_path TEXT,
|
||||
notes TEXT DEFAULT '',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_assets_category ON assets(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_csb_status ON cantaloupe_sync_batches(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_csb_created ON cantaloupe_sync_batches(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_checkins_asset_id ON checkins(asset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_checkins_created_at ON checkins(created_at);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_assets_machine_id ON assets(machine_id);
|
||||
"""
|
||||
|
||||
# ── 4. Map merged record to asset row ────────────────────────────────────
|
||||
def get_machine_id(rec):
|
||||
"""Get machine_id, falling back from msfs to canteen."""
|
||||
mid = rec.get("msfs_machine_id")
|
||||
if mid:
|
||||
return str(mid).strip()
|
||||
mid = rec.get("canteen_machine_id")
|
||||
if mid:
|
||||
return str(mid).strip()
|
||||
return ""
|
||||
|
||||
def map_record_to_asset(rec):
|
||||
"""
|
||||
Map a merged record's fields to the assets table columns.
|
||||
Returns a dict keyed by column name.
|
||||
"""
|
||||
# Primary identifiers
|
||||
machine_id = get_machine_id(rec)
|
||||
name = rec.get("msfs_name") or rec.get("canteen_name") or ""
|
||||
|
||||
# Category / status
|
||||
category = rec.get("canteen_category") or "Other"
|
||||
status = rec.get("canteen_status") or "active"
|
||||
|
||||
# Make / model / serial
|
||||
make = rec.get("msfs_manufacturer") or ""
|
||||
model = rec.get("msfs_dex_model") or ""
|
||||
serial_number = rec.get("serial_number") or ""
|
||||
|
||||
# GPS — prefer msfs, fall back to canteen, then account
|
||||
latitude = rec.get("msfs_latitude")
|
||||
longitude = rec.get("msfs_longitude")
|
||||
if latitude is None or longitude is None:
|
||||
latitude = rec.get("canteen_latitude") or latitude
|
||||
longitude = rec.get("canteen_longitude") or longitude
|
||||
|
||||
# Address — from account if available
|
||||
address = ""
|
||||
account = rec.get("account")
|
||||
if account and account.get("address"):
|
||||
addr_parts = [account["address"]]
|
||||
if account.get("city"):
|
||||
addr_parts.append(account["city"])
|
||||
if account.get("state"):
|
||||
addr_parts.append(account["state"])
|
||||
if account.get("zip"):
|
||||
addr_parts.append(account["zip"])
|
||||
address = ", ".join(addr_parts)
|
||||
if not address:
|
||||
address = rec.get("canteen_address") or ""
|
||||
|
||||
# Canteen fields
|
||||
customer_id = rec.get("canteen_customer_id")
|
||||
disney_park = rec.get("canteen_disney_park")
|
||||
is_disney = rec.get("canteen_is_disney") or 0
|
||||
dex_report_date = rec.get("canteen_dex_report_date")
|
||||
install_date = rec.get("canteen_install_date") or rec.get("msfs_install_date")
|
||||
building_name = rec.get("canteen_building_name")
|
||||
building_number = rec.get("canteen_building_number")
|
||||
floor = rec.get("canteen_floor") or ""
|
||||
room = rec.get("canteen_room") or ""
|
||||
geofence_radius = rec.get("canteen_geofence_radius") or 50
|
||||
company = rec.get("canteen_company") or ""
|
||||
description = rec.get("canteen_description") or ""
|
||||
deployed = rec.get("canteen_deployed")
|
||||
pulled_date = rec.get("canteen_pulled_date")
|
||||
location_area = rec.get("canteen_location_area") or ""
|
||||
place = rec.get("canteen_place") or ""
|
||||
location_id = rec.get("canteen_location_id")
|
||||
|
||||
# MSFS fields
|
||||
connect_id = rec.get("msfs_connect_id") or ""
|
||||
canteen_connect_guid = rec.get("msfs_canteen_connect_guid") or ""
|
||||
manufacturer = rec.get("msfs_manufacturer") or ""
|
||||
equipment_id = rec.get("msfs_equipment_id") or ""
|
||||
|
||||
return {
|
||||
"machine_id": machine_id,
|
||||
"serial_number": serial_number,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"category": category,
|
||||
"status": status,
|
||||
"make": make,
|
||||
"model": model,
|
||||
"address": address,
|
||||
"building_name": building_name or "",
|
||||
"building_number": building_number or "",
|
||||
"floor": floor,
|
||||
"room": room,
|
||||
"customer_id": customer_id,
|
||||
"location_id": location_id,
|
||||
"latitude": latitude,
|
||||
"longitude": longitude,
|
||||
"geofence_radius_meters": geofence_radius,
|
||||
"disney_park": disney_park,
|
||||
"is_disney": is_disney,
|
||||
"dex_report_date": dex_report_date,
|
||||
"install_date": install_date,
|
||||
"deployed": deployed,
|
||||
"pulled_date": pulled_date,
|
||||
"company": company,
|
||||
"location_area": location_area,
|
||||
"place": place,
|
||||
"connect_id": connect_id,
|
||||
"canteen_connect_guid": canteen_connect_guid,
|
||||
"manufacturer": manufacturer,
|
||||
"equipment_id": equipment_id,
|
||||
}
|
||||
|
||||
# ── 5. Insert records into DB ────────────────────────────────────────────
|
||||
def create_db(records):
|
||||
"""Create a fresh assets.db with schema and import records."""
|
||||
# Remove existing
|
||||
if os.path.exists(DB_PATH):
|
||||
os.remove(DB_PATH)
|
||||
print("[DB] Removed existing assets.db")
|
||||
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
print("[DB] Created fresh assets.db with full schema")
|
||||
|
||||
# Insert default categories used by the data
|
||||
cursor = conn.cursor()
|
||||
categories_seen = set()
|
||||
for rec in records:
|
||||
cat = rec.get("canteen_category") or "Other"
|
||||
categories_seen.add(cat)
|
||||
for cat in sorted(categories_seen):
|
||||
cursor.execute("INSERT OR IGNORE INTO categories (name) VALUES (?)", (cat,))
|
||||
print(f"[CATEGORIES] Inserted {len(categories_seen)} categories")
|
||||
|
||||
# Insert customers from the data
|
||||
customers_seen = {}
|
||||
for rec in records:
|
||||
if rec.get("customer") and rec["customer"].get("name"):
|
||||
cust = rec["customer"]
|
||||
cid = cust.get("id")
|
||||
cname = cust.get("name")
|
||||
if cid and cname and cid not in customers_seen:
|
||||
customers_seen[cid] = cname
|
||||
for cid, cname in sorted(customers_seen.items()):
|
||||
cursor.execute(
|
||||
"INSERT OR IGNORE INTO customers (id, name) VALUES (?, ?)",
|
||||
(cid, cname),
|
||||
)
|
||||
print(f"[CUSTOMERS] Inserted {len(customers_seen)} customers")
|
||||
|
||||
# Insert locations from the data
|
||||
locations_seen = {}
|
||||
for rec in records:
|
||||
if rec.get("location") and rec["location"].get("name"):
|
||||
loc = rec["location"]
|
||||
lid = loc.get("id")
|
||||
lname = loc.get("name")
|
||||
if lid and lname and lid not in locations_seen:
|
||||
locations_seen[lid] = {
|
||||
"name": lname,
|
||||
"address": loc.get("address") or "",
|
||||
"customer_id": rec.get("canteen_customer_id"),
|
||||
}
|
||||
for lid, loc_data in sorted(locations_seen.items()):
|
||||
cursor.execute(
|
||||
"INSERT OR IGNORE INTO locations (id, name, address, customer_id) VALUES (?, ?, ?, ?)",
|
||||
(lid, loc_data["name"], loc_data["address"], loc_data["customer_id"]),
|
||||
)
|
||||
print(f"[LOCATIONS] Inserted {len(locations_seen)} locations")
|
||||
|
||||
# Insert default users
|
||||
default_users = [
|
||||
("admin", hash_password("admin123"), "admin"),
|
||||
("tech", hash_password("tech123"), "technician"),
|
||||
]
|
||||
for username, pw_hash, role in default_users:
|
||||
cursor.execute(
|
||||
"INSERT INTO users (username, password_hash, role) VALUES (?, ?, ?)",
|
||||
(username, pw_hash, role),
|
||||
)
|
||||
print(f"[USERS] Inserted {len(default_users)} default users")
|
||||
|
||||
# Insert assets — deduplicate by machine_id, preferring "joined" over "msfs_only"
|
||||
asset_cols = [
|
||||
"machine_id", "serial_number", "name", "description", "category",
|
||||
"status", "make", "model", "address", "building_name",
|
||||
"building_number", "floor", "room", "customer_id", "location_id",
|
||||
"latitude", "longitude", "geofence_radius_meters", "disney_park",
|
||||
"is_disney", "dex_report_date", "install_date", "deployed",
|
||||
"pulled_date", "company", "location_area", "place", "connect_id",
|
||||
"canteen_connect_guid", "manufacturer", "equipment_id",
|
||||
]
|
||||
placeholders = ", ".join(["?"] * len(asset_cols))
|
||||
col_names = ", ".join(asset_cols)
|
||||
|
||||
# Priority: joined > msfs_only > canteen_only
|
||||
match_priority = {"joined": 0, "msfs_only": 1, "canteen_only": 2}
|
||||
|
||||
deduped = {} # machine_id -> (priority, row)
|
||||
no_id = 0
|
||||
for rec in records:
|
||||
row = map_record_to_asset(rec)
|
||||
mid = row["machine_id"]
|
||||
if not mid:
|
||||
no_id += 1
|
||||
continue
|
||||
priority = match_priority.get(rec.get("match", ""), 99)
|
||||
if mid not in deduped or priority < deduped[mid][0]:
|
||||
deduped[mid] = (priority, row)
|
||||
|
||||
inserted = 0
|
||||
for mid, (priority, row) in deduped.items():
|
||||
values = [row[c] for c in asset_cols]
|
||||
cursor.execute(
|
||||
f"INSERT INTO assets ({col_names}) VALUES ({placeholders})",
|
||||
values,
|
||||
)
|
||||
inserted += 1
|
||||
|
||||
total_raw = len(records)
|
||||
total_deduped = len(deduped)
|
||||
skipped = total_raw - total_deduped - no_id
|
||||
print(f"\n[ASSETS] Raw records: {total_raw}")
|
||||
print(f"[ASSETS] No machine_id: {no_id}")
|
||||
print(f"[ASSETS] Duplicates removed: {skipped}")
|
||||
print(f"[ASSETS] Inserted: {inserted}")
|
||||
print(f"[ASSETS] Total in DB: {inserted}")
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Count
|
||||
asset_count = cursor.execute("SELECT COUNT(*) FROM assets").fetchone()[0]
|
||||
gps_count = cursor.execute(
|
||||
"SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL AND longitude IS NOT NULL"
|
||||
).fetchone()[0]
|
||||
cat_dist = cursor.execute(
|
||||
"SELECT category, COUNT(*) FROM assets GROUP BY category ORDER BY COUNT(*) DESC"
|
||||
).fetchall()
|
||||
status_dist = cursor.execute(
|
||||
"SELECT status, COUNT(*) FROM assets GROUP BY status ORDER BY COUNT(*) DESC"
|
||||
).fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
print(f"[ASSETS] With GPS coords: {gps_count}")
|
||||
print(f"\n[ASSETS] Category distribution:")
|
||||
for cat, cnt in cat_dist:
|
||||
print(f" {cat}: {cnt}")
|
||||
print(f"\n[ASSETS] Status distribution:")
|
||||
for st, cnt in status_dist:
|
||||
print(f" {st}: {cnt}")
|
||||
|
||||
return asset_count, gps_count
|
||||
|
||||
# ── 6. Verification ──────────────────────────────────────────────────────
|
||||
def verify_db():
|
||||
"""Open the fresh DB and run basic checks."""
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
print("\n─── VERIFICATION ───")
|
||||
|
||||
# Table list
|
||||
tables = cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
).fetchall()
|
||||
print(f"Tables ({len(tables)}): {', '.join(t['name'] for t in tables)}")
|
||||
|
||||
# User count
|
||||
user_count = cursor.execute("SELECT COUNT(*) FROM users").fetchone()[0]
|
||||
print(f"Users: {user_count}")
|
||||
users = cursor.execute("SELECT username, role FROM users").fetchall()
|
||||
for u in users:
|
||||
print(f" - {u['username']} ({u['role']})")
|
||||
|
||||
# Asset count
|
||||
asset_count = cursor.execute("SELECT COUNT(*) FROM assets").fetchone()[0]
|
||||
gps_count = cursor.execute(
|
||||
"SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL AND longitude IS NOT NULL"
|
||||
).fetchone()[0]
|
||||
print(f"Assets: {asset_count}")
|
||||
print(f"Assets with GPS: {gps_count}")
|
||||
|
||||
# Sample from each match type
|
||||
if asset_count > 0:
|
||||
print("\nSample records:")
|
||||
sample = cursor.execute(
|
||||
"SELECT machine_id, name, category, status, latitude, longitude FROM assets LIMIT 5"
|
||||
).fetchall()
|
||||
for s in sample:
|
||||
gps_tag = f"({s['latitude']}, {s['longitude']})" if s['latitude'] else "NO GPS"
|
||||
print(f" {s['machine_id']}: {s['name']} [{s['category']}] {gps_tag}")
|
||||
|
||||
# Check index count
|
||||
idx_count = cursor.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'"
|
||||
).fetchone()[0]
|
||||
print(f"Indexes: {idx_count}")
|
||||
|
||||
# DB size
|
||||
db_size = os.path.getsize(DB_PATH) / (1024 * 1024)
|
||||
print(f"DB size: {db_size:.1f} MB")
|
||||
|
||||
conn.close()
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────
|
||||
def main():
|
||||
print("═══ MSFS MERGED DATA IMPORT ═══\n")
|
||||
|
||||
# 1. Backup
|
||||
backup_path = backup_db()
|
||||
|
||||
# 2. Read data
|
||||
records = read_merged_data()
|
||||
|
||||
# 3. Create DB and import
|
||||
asset_count, gps_count = create_db(records)
|
||||
|
||||
# 4. Verify
|
||||
verify_db()
|
||||
|
||||
print(f"\n═══ DONE ═══")
|
||||
if backup_path:
|
||||
print(f"Backup: {backup_path}")
|
||||
print(f"New DB: {DB_PATH} ({asset_count} assets, {gps_count} with GPS)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import Seed (mycantaloupe.com) Excel data into the Canteen Asset Tracker.
|
||||
|
||||
Steps:
|
||||
1. Read Machine List(8).xlsx — 62 columns of Seed export
|
||||
2. Match rows by Asset ID → assets.machine_id
|
||||
3. Add key columns to assets table
|
||||
4. Create seed_data table with full JSON per asset
|
||||
5. Update matched assets, report match stats
|
||||
|
||||
Usage:
|
||||
python3 import_seed.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, date
|
||||
|
||||
import openpyxl
|
||||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────────
|
||||
PROJECT_DIR = os.path.expanduser("~/projects/canteen-asset-tracker")
|
||||
DB_PATH = os.path.join(PROJECT_DIR, "assets.db")
|
||||
EXCEL_PATH = os.path.expanduser(
|
||||
"/home/oplabs/.hermes/profiles/coder/cache/documents/doc_305e40fb92dd_Machine List(8).xlsx"
|
||||
)
|
||||
|
||||
# ── Columns to extract as first-class DB columns ──────────────────────────
|
||||
# (db_column, excel_header, type)
|
||||
KEY_COLUMNS = [
|
||||
("device", "Device", "TEXT DEFAULT ''"),
|
||||
("seed_location", "Location", "TEXT DEFAULT ''"),
|
||||
("seed_city", "City", "TEXT DEFAULT ''"),
|
||||
("state", "State", "TEXT DEFAULT ''"),
|
||||
("postal_code", "Postal Code", "TEXT DEFAULT ''"),
|
||||
("route_name", "Route", "TEXT DEFAULT ''"),
|
||||
("subroute_name", "Subroute", "TEXT DEFAULT ''"),
|
||||
("seed_class", "Class", "TEXT DEFAULT ''"),
|
||||
("customer_name", "Customer", "TEXT DEFAULT ''"),
|
||||
("management_company", "Management Company", "TEXT DEFAULT ''"),
|
||||
("branch", "Branch", "TEXT DEFAULT ''"),
|
||||
("location_code", "Location Code", "TEXT DEFAULT ''"),
|
||||
("customer_code", "Customer Code", "TEXT DEFAULT ''"),
|
||||
("barcode", "Barcode", "TEXT DEFAULT ''"),
|
||||
("has_cashless", "Has Cashless", "TEXT DEFAULT ''"),
|
||||
("phone", "Phone", "TEXT DEFAULT ''"),
|
||||
("fax", "Fax", "TEXT DEFAULT ''"),
|
||||
("email", "Email", "TEXT DEFAULT ''"),
|
||||
("asset_family", "Asset Family", "TEXT DEFAULT ''"),
|
||||
("business_type", "Business Type", "TEXT DEFAULT ''"),
|
||||
("primary_consumer_type","Primary Consumer Type","TEXT DEFAULT ''"),
|
||||
("machine_branding", "Machine Branding", "TEXT DEFAULT ''"),
|
||||
("valid_address", "Valid Address", "TEXT DEFAULT ''"),
|
||||
("non_revenue", "Non-Revenue", "TEXT DEFAULT ''"),
|
||||
("alerts", "Alerts", "TEXT DEFAULT ''"),
|
||||
("coil_alerts", "Coil Alerts", "INTEGER DEFAULT 0"),
|
||||
("product_alerts", "Product Alerts", "INTEGER DEFAULT 0"),
|
||||
("daily_avg_sales", "Daily Average Sales", "REAL DEFAULT NULL"),
|
||||
("monthly_sales", "Monthly Sales", "REAL DEFAULT NULL"),
|
||||
("yearly_sales", "Yearly Sales", "REAL DEFAULT NULL"),
|
||||
("today_sales", "Today Sales", "REAL DEFAULT NULL"),
|
||||
("yesterday_sales", "Yesterday Sales", "REAL DEFAULT NULL"),
|
||||
("weekly_sales", "Weekly Sales", "REAL DEFAULT NULL"),
|
||||
("sales_restock", "Sales (Restock)", "REAL DEFAULT NULL"),
|
||||
("last_restock", "Last Restock", "TEXT DEFAULT ''"),
|
||||
("days_since_restock", "Days Since Restock", "INTEGER DEFAULT NULL"),
|
||||
("last_contact_time", "Last Contact Time", "TEXT DEFAULT ''"),
|
||||
("last_dex_report_time", "Last Dex Report Time","TEXT DEFAULT ''"),
|
||||
("last_inventory", "Last Inventory", "TEXT DEFAULT ''"),
|
||||
("prepick_group", "Prepick Group", "TEXT DEFAULT ''"),
|
||||
("added_date", "Added Date", "TEXT DEFAULT ''"),
|
||||
("purchase_date", "Purchase Date", "TEXT DEFAULT ''"),
|
||||
("purchase_price", "Purchase Price", "REAL DEFAULT NULL"),
|
||||
("depreciation_years", "Depreciation Years", "REAL DEFAULT NULL"),
|
||||
("cash_discount", "Cash Discount", "REAL DEFAULT NULL"),
|
||||
("tax_jurisdiction", "Tax Jurisdiction", "TEXT DEFAULT ''"),
|
||||
("commission_plan", "Commission Plan", "TEXT DEFAULT ''"),
|
||||
("acquirer_from", "Acquired From", "TEXT DEFAULT ''"),
|
||||
("changer_par", "Changer Par", "TEXT DEFAULT ''"),
|
||||
("machine_management_code","Machine Management Code","TEXT DEFAULT ''"),
|
||||
]
|
||||
|
||||
# ── All 62 columns for the seed_data JSON dump ────────────────────────────
|
||||
ALL_SEED_FIELDS = [
|
||||
"Device", "Location", "Asset ID", "Place", "Type",
|
||||
"City", "Address", "Last Contact Time", "Last Dex Report Time",
|
||||
"Last Restock", "Coil Alerts", "Product Alerts", "Sales (Restock)",
|
||||
"Daily Average Sales", "Today Sales", "Yesterday Sales", "Weekly Sales",
|
||||
"Monthly Sales", "Yearly Sales", "Days Since Restock", "Prepick Group",
|
||||
"Customer", "Management Company", "Management Account",
|
||||
"Machine Management Code", "Route", "Subroute", "Changer Par",
|
||||
"Acquired From", "Purchase Date", "Purchase Price", "Depreciation Years",
|
||||
"Post-Depreciation Monthly Cost", "State", "Postal Code", "Deployed",
|
||||
"Pulled Date", "Serial Number", "Class", "Make", "Model",
|
||||
"Cash Discount", "Tax Jurisdiction", "Commission Plan", "Barcode",
|
||||
"Non-Revenue", "Added Date", "Phone", "Fax", "Email", "Has Cashless",
|
||||
"Branch", "Location Code", "Customer Code", "Last Inventory",
|
||||
"Asset Family", "Status", "Alerts", "Valid Address",
|
||||
"Business Type", "Primary Consumer Type", "Machine Branding",
|
||||
]
|
||||
|
||||
|
||||
# ── Schema migration helpers ──────────────────────────────────────────────
|
||||
def get_db_columns(conn):
|
||||
cursor = conn.execute("PRAGMA table_info(assets)")
|
||||
return {row[1] for row in cursor.fetchall()}
|
||||
|
||||
|
||||
def add_column_if_missing(conn, col_name, col_type):
|
||||
existing = get_db_columns(conn)
|
||||
if col_name not in existing:
|
||||
sql = f"ALTER TABLE assets ADD COLUMN {col_name} {col_type}"
|
||||
conn.execute(sql)
|
||||
print(f" Added column: {col_name} {col_type}")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def create_seed_data_table(conn):
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS seed_data (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
asset_id INTEGER NOT NULL REFERENCES assets(id) ON DELETE CASCADE,
|
||||
raw_data TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE(asset_id)
|
||||
)
|
||||
""")
|
||||
print(" seed_data table ready")
|
||||
# Add index
|
||||
conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_seed_data_asset_id
|
||||
ON seed_data(asset_id)
|
||||
""")
|
||||
|
||||
|
||||
# ── Parse Excel ──────────────────────────────────────────────────────────
|
||||
def read_excel():
|
||||
"""Return (header_map, rows) where header_map is ExcelHeader->col_index."""
|
||||
wb = openpyxl.load_workbook(EXCEL_PATH, data_only=True)
|
||||
ws = wb.active
|
||||
headers = {}
|
||||
for col in range(1, ws.max_column + 1):
|
||||
val = ws.cell(row=1, column=col).value
|
||||
if val is not None:
|
||||
headers[str(val).strip()] = col
|
||||
|
||||
rows = []
|
||||
for row_idx in range(2, ws.max_row + 1):
|
||||
row_data = {}
|
||||
for hdr, col in headers.items():
|
||||
val = ws.cell(row=row_idx, column=col).value
|
||||
row_data[hdr] = val
|
||||
row_data["_row"] = row_idx
|
||||
rows.append(row_data)
|
||||
|
||||
return headers, rows
|
||||
|
||||
|
||||
# ── Main import ──────────────────────────────────────────────────────────
|
||||
def main():
|
||||
print("═══ SEED DATA IMPORT ═══\n")
|
||||
|
||||
# 1. Connect to DB
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
print(f"[DB] Connected: {DB_PATH}")
|
||||
|
||||
# 2. Schema migration
|
||||
print("\n[MIGRATE] Adding new asset columns if missing...")
|
||||
for col_name, excel_header, col_type in KEY_COLUMNS:
|
||||
add_column_if_missing(conn, col_name, col_type)
|
||||
|
||||
print("\n[MIGRATE] Creating seed_data table...")
|
||||
create_seed_data_table(conn)
|
||||
conn.commit()
|
||||
|
||||
# 3. Read Excel
|
||||
print(f"\n[READ] Reading {EXCEL_PATH}...")
|
||||
headers, rows = read_excel()
|
||||
print(f"[READ] Found {len(headers)} columns, {len(rows)} data rows")
|
||||
|
||||
# 4. Pre-build machine_id lookup
|
||||
cursor = conn.execute("SELECT id, machine_id FROM assets")
|
||||
machine_map = {} # machine_id -> asset_id
|
||||
for row in cursor.fetchall():
|
||||
machine_map[str(row["machine_id"]).strip()] = row["id"]
|
||||
|
||||
print(f"[MATCH] DB has {len(machine_map)} assets to match against")
|
||||
|
||||
# 5. Import
|
||||
matched = 0
|
||||
skipped_no_id = 0
|
||||
unmatched = 0
|
||||
key_field_updates = 0
|
||||
json_inserts = 0
|
||||
|
||||
for row in rows:
|
||||
asset_id_val = row.get("Asset ID")
|
||||
if asset_id_val is None or str(asset_id_val).strip() == "":
|
||||
skipped_no_id += 1
|
||||
continue
|
||||
|
||||
asset_id_str = str(asset_id_val).strip()
|
||||
db_asset_id = machine_map.get(asset_id_str)
|
||||
|
||||
if db_asset_id is None:
|
||||
unmatched += 1
|
||||
continue
|
||||
|
||||
# Update key columns on assets table
|
||||
update_parts = []
|
||||
update_vals = []
|
||||
for col_name, excel_header, col_type in KEY_COLUMNS:
|
||||
val = row.get(excel_header)
|
||||
# Handle None → NULL for SQL
|
||||
update_parts.append(f"{col_name} = ?")
|
||||
if val is None:
|
||||
update_vals.append(None)
|
||||
else:
|
||||
if "INTEGER" in col_type:
|
||||
try:
|
||||
update_vals.append(int(val))
|
||||
except (ValueError, TypeError):
|
||||
update_vals.append(0)
|
||||
elif "REAL" in col_type:
|
||||
try:
|
||||
update_vals.append(float(val))
|
||||
except (ValueError, TypeError):
|
||||
update_vals.append(None)
|
||||
else:
|
||||
update_vals.append(str(val))
|
||||
|
||||
update_vals.append(db_asset_id)
|
||||
conn.execute(
|
||||
f"UPDATE assets SET {', '.join(update_parts)} WHERE id = ?",
|
||||
update_vals,
|
||||
)
|
||||
key_field_updates += 1
|
||||
|
||||
# Insert/upsert seed_data JSON
|
||||
raw_data = {}
|
||||
for hdr in ALL_SEED_FIELDS:
|
||||
val = row.get(hdr)
|
||||
if val is not None:
|
||||
raw_data[hdr] = val
|
||||
|
||||
# Convert non-serializable types (datetime, etc.) to strings
|
||||
def _serialize(v):
|
||||
if isinstance(v, (datetime, date)):
|
||||
return v.isoformat()
|
||||
return v
|
||||
raw_json = json.dumps(raw_data, default=_serialize)
|
||||
conn.execute(
|
||||
"""INSERT OR REPLACE INTO seed_data (asset_id, raw_data, imported_at)
|
||||
VALUES (?, ?, datetime('now'))""",
|
||||
(db_asset_id, raw_json),
|
||||
)
|
||||
json_inserts += 1
|
||||
matched += 1
|
||||
|
||||
conn.commit()
|
||||
|
||||
# 6. Stats
|
||||
print(f"\n═══ RESULTS ═══")
|
||||
print(f"Seed rows: {len(rows)}")
|
||||
print(f"Matched (↔ assets): {matched}")
|
||||
print(f"Unmatched (no asset): {unmatched}")
|
||||
print(f"Rows without Asset ID: {skipped_no_id}")
|
||||
print(f"Key field updates: {key_field_updates}")
|
||||
print(f"JSON blobs inserted: {json_inserts}")
|
||||
|
||||
# Re-count assets with seed data
|
||||
seed_count = conn.execute(
|
||||
"SELECT COUNT(*) FROM seed_data"
|
||||
).fetchone()[0]
|
||||
print(f"\nseed_data table: {seed_count} rows")
|
||||
|
||||
# Show DB size
|
||||
db_size = os.path.getsize(DB_PATH) / (1024 * 1024)
|
||||
print(f"DB size: {db_size:.1f} MB")
|
||||
|
||||
conn.close()
|
||||
print("\n═══ DONE ═══")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Regular → Executable
+7
-13
@@ -1,16 +1,10 @@
|
||||
#!/bin/bash
|
||||
# Restart canteen-asset-tracker server
|
||||
cd /home/oplabs/projects/canteen-asset-tracker
|
||||
|
||||
# Kill existing server
|
||||
pkill -f "python.*server.py" 2>/dev/null || true
|
||||
pkill -f "uvicorn.*server:app.*8901" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Activate venv and start
|
||||
source .venv/bin/activate
|
||||
nohup python server.py > /tmp/canteen-server.log 2>&1 &
|
||||
echo "Server PID: $!"
|
||||
sleep 3
|
||||
|
||||
# Check health
|
||||
curl -s http://localhost:8901/health 2>/dev/null || curl -s http://localhost:8900/health 2>/dev/null || echo "Health check failed"
|
||||
python -m uvicorn server:app --host 0.0.0.0 --port 8901 \
|
||||
--ssl-keyfile key.pem --ssl-certfile cert.pem --log-level info \
|
||||
> /tmp/canteen-server.log 2>&1 &
|
||||
echo "PID=$!"
|
||||
sleep 2
|
||||
curl -sk https://localhost:8901/health 2>/dev/null
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Apply GPS coordinates from multiple sources to canteen assets DB.
|
||||
|
||||
Sources (priority order):
|
||||
1. Booking GPS (MSFS technician check-in) — most reliable
|
||||
2. Photo EXIF GPS (OCR'd technician photos)
|
||||
3. exif-test-dev GPS (iPhone uploads)
|
||||
|
||||
Usage:
|
||||
python3 scripts/apply_gps_sources.py # dry-run
|
||||
python3 scripts/apply_gps_sources.py --apply # write to DB
|
||||
python3 scripts/apply_gps_sources.py --force # overwrite existing GPS too
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ASSET_DB = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
MSFS_DATA_DIR = "/home/oplabs/projects/ms-field-service-extraction/web/static/data"
|
||||
MSFS_BACKUP_DB = "/home/oplabs/projects/ms-field-service-extraction/data-samples/msfs_backup/fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db"
|
||||
|
||||
|
||||
def load_merged_assets() -> dict:
|
||||
"""Load merged MSFS<->canteen asset mappings. Returns guid->machine_id."""
|
||||
with open(f"{MSFS_DATA_DIR}/merged-assets.json") as f:
|
||||
merged = json.load(f)
|
||||
guid_to_mid = {}
|
||||
for a in merged["assets"]:
|
||||
cmid = a.get("canteen_machine_id")
|
||||
guid = a.get("msfs_canteen_connect_guid", "")
|
||||
if cmid and guid:
|
||||
guid_to_mid[guid.upper()] = cmid
|
||||
return guid_to_mid
|
||||
|
||||
|
||||
def get_booking_gps(guid_to_mid: dict) -> dict:
|
||||
"""Extract booking GPS from MSFS backup, grouped by machine_id."""
|
||||
db = sqlite3.connect(MSFS_BACKUP_DB)
|
||||
|
||||
# Work order -> customer asset
|
||||
wo_to_asset = {}
|
||||
for w in db.execute(
|
||||
'SELECT "msdyn_workorderid", "msdyn_customerasset!id" FROM "msdyn_workorder" WHERE "msdyn_customerasset!id" IS NOT NULL'
|
||||
):
|
||||
wo_to_asset[w[0]] = w[1]
|
||||
|
||||
# Customer asset -> GUID
|
||||
asset_info = {}
|
||||
for c in db.execute(
|
||||
'SELECT "msdyn_customerassetid", "hsl_canteenconnectguid" FROM "msdyn_customerasset"'
|
||||
):
|
||||
asset_info[c[0]] = (c[1] or "").upper() if c[1] else ""
|
||||
|
||||
# Collect booking GPS points
|
||||
booking_by_mid = defaultdict(list)
|
||||
for r in db.execute(
|
||||
"""
|
||||
SELECT b."msdyn_latitude", b."msdyn_longitude", b."msdyn_workorder!id"
|
||||
FROM "bookableresourcebooking" b
|
||||
WHERE b."msdyn_latitude" IS NOT NULL AND b."msdyn_longitude" IS NOT NULL
|
||||
"""
|
||||
):
|
||||
lat, lon, wo_id = r
|
||||
if not wo_id or wo_id not in wo_to_asset:
|
||||
continue
|
||||
asset_id = wo_to_asset[wo_id]
|
||||
if not asset_id or asset_id not in asset_info:
|
||||
continue
|
||||
guid = asset_info[asset_id]
|
||||
if guid in guid_to_mid:
|
||||
mid = guid_to_mid[guid]
|
||||
booking_by_mid[mid].append((lat, lon))
|
||||
|
||||
# Average per machine
|
||||
result = {}
|
||||
for mid, points in booking_by_mid.items():
|
||||
result[mid] = {
|
||||
"lat": sum(p[0] for p in points) / len(points),
|
||||
"lon": sum(p[1] for p in points) / len(points),
|
||||
"source": "booking",
|
||||
"count": len(points),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def get_photo_gps() -> dict:
|
||||
"""Extract GPS from OCR'd technician photos."""
|
||||
try:
|
||||
with open(f"{MSFS_DATA_DIR}/ocr_results.json") as f:
|
||||
ocr = json.load(f)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
photo_by_mid = defaultdict(list)
|
||||
for k, v in ocr.items():
|
||||
if v.get("has_gps") and v.get("machine_id") and v.get("gps_lat") and v.get("gps_lon"):
|
||||
mid = v["machine_id"]
|
||||
# Normalize: strip leading zeros from last part
|
||||
parts = mid.split("-")
|
||||
last_part = parts[-1]
|
||||
if last_part.startswith("0") and len(last_part) > 5:
|
||||
last_part = last_part.lstrip("0")
|
||||
photo_by_mid[last_part].append((v["gps_lat"], v["gps_lon"]))
|
||||
|
||||
result = {}
|
||||
for mid, points in photo_by_mid.items():
|
||||
result[mid] = {
|
||||
"lat": sum(p[0] for p in points) / len(points),
|
||||
"lon": sum(p[1] for p in points) / len(points),
|
||||
"source": "photo",
|
||||
"count": len(points),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def get_exif_test_gps() -> dict:
|
||||
"""Extract GPS from exif-test-dev iPhone uploads."""
|
||||
exif_dev_db = str(
|
||||
Path.home() / "projects" / "exif-test-dev" / "photos.db"
|
||||
)
|
||||
if not Path(exif_dev_db).exists():
|
||||
return {}
|
||||
|
||||
db = sqlite3.connect(exif_dev_db)
|
||||
result = {}
|
||||
for r in db.execute(
|
||||
"SELECT machine_id, gps_lat, gps_lng FROM photos WHERE machine_id IS NOT NULL AND gps_lat IS NOT NULL"
|
||||
):
|
||||
mid, lat, lng = r
|
||||
if mid not in result:
|
||||
result[mid] = {"points": [], "source": "exif_dev"}
|
||||
result[mid]["points"].append((lat, lng))
|
||||
|
||||
averaged = {}
|
||||
for mid, data in result.items():
|
||||
pts = data["points"]
|
||||
averaged[mid] = {
|
||||
"lat": sum(p[0] for p in pts) / len(pts),
|
||||
"lon": sum(p[1] for p in pts) / len(pts),
|
||||
"source": "exif_dev",
|
||||
"count": len(pts),
|
||||
}
|
||||
return averaged
|
||||
|
||||
|
||||
def get_seed_gps(conn: sqlite3.Connection) -> dict:
|
||||
"""Extract GPS from Cantaloupe seed data if available."""
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"SELECT machine_id, json_extract(data, '$.Latitude'), json_extract(data, '$.Longitude') "
|
||||
"FROM seed_data "
|
||||
"WHERE json_extract(data, '$.Latitude') IS NOT NULL "
|
||||
"AND json_extract(data, '$.Longitude') IS NOT NULL"
|
||||
)
|
||||
result = {}
|
||||
for row in cur.fetchall():
|
||||
mid, lat, lon = row
|
||||
if lat and lon:
|
||||
try:
|
||||
result[mid] = {
|
||||
"lat": float(lat),
|
||||
"lon": float(lon),
|
||||
"source": "seed",
|
||||
"count": 1,
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return result
|
||||
except sqlite3.OperationalError:
|
||||
return {}
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
force_update = "--force" in sys.argv
|
||||
mode = "DRY RUN (no writes)" if not apply_changes else "APPLYING changes"
|
||||
|
||||
print(f"Assets DB: {ASSET_DB}")
|
||||
print(f"Mode: {mode}")
|
||||
if force_update:
|
||||
print(" (--force: will overwrite existing GPS coordinates too)")
|
||||
print()
|
||||
|
||||
# 1. Load all GPS sources
|
||||
print("Loading GPS sources...")
|
||||
guid_to_mid = load_merged_assets()
|
||||
print(f" GUID→machine_id mappings: {len(guid_to_mid)}")
|
||||
|
||||
booking_gps = get_booking_gps(guid_to_mid)
|
||||
print(f" Booking GPS: {len(booking_gps)} machine IDs")
|
||||
|
||||
photo_gps = get_photo_gps()
|
||||
print(f" Photo EXIF GPS: {len(photo_gps)} machine IDs")
|
||||
|
||||
exif_gps = get_exif_test_gps()
|
||||
print(f" EXIF test GPS: {len(exif_gps)} machine IDs")
|
||||
|
||||
# Connect to assets DB
|
||||
conn = sqlite3.connect(ASSET_DB)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=OFF")
|
||||
|
||||
seed_gps = get_seed_gps(conn)
|
||||
print(f" Seed (Cantaloupe) GPS: {len(seed_gps)} machine IDs")
|
||||
|
||||
# 2. Get current GPS state from assets DB
|
||||
cur = conn.execute("SELECT machine_id, latitude, longitude, name, company FROM assets")
|
||||
existing = {}
|
||||
for r in cur.fetchall():
|
||||
existing[r[0]] = {"lat": r[1], "lon": r[2], "name": r[3], "company": r[4]}
|
||||
|
||||
existing_count = sum(1 for v in existing.values() if v["lat"] is not None)
|
||||
print(f"\nCurrent assets with GPS: {existing_count} / {len(existing)}")
|
||||
|
||||
# 3. Prioritize and merge sources
|
||||
all_mids = set()
|
||||
all_mids.update(booking_gps.keys())
|
||||
all_mids.update(photo_gps.keys())
|
||||
all_mids.update(exif_gps.keys())
|
||||
all_mids.update(seed_gps.keys())
|
||||
|
||||
updates = [] # (lat, lon, source, machine_id)
|
||||
skipped_existing = []
|
||||
no_match = []
|
||||
|
||||
for mid in sorted(all_mids):
|
||||
# Determine best GPS: booking > photo > exif_dev > seed
|
||||
best = None
|
||||
for src_dict, src_name in [
|
||||
(booking_gps, "booking"),
|
||||
(photo_gps, "photo"),
|
||||
(exif_gps, "exif_dev"),
|
||||
(seed_gps, "seed"),
|
||||
]:
|
||||
if mid in src_dict:
|
||||
best = src_dict[mid]
|
||||
break
|
||||
|
||||
if not best:
|
||||
continue
|
||||
|
||||
# Check if asset exists in DB
|
||||
if mid not in existing:
|
||||
no_match.append(mid)
|
||||
continue
|
||||
|
||||
existing_gps = existing[mid]
|
||||
has_gps = existing_gps["lat"] is not None and existing_gps["lon"] is not None
|
||||
|
||||
if has_gps and not force_update:
|
||||
# Code for checking distance removed — just skip
|
||||
skipped_existing.append((mid, existing_gps["lat"], existing_gps["lon"], best["lat"], best["lon"]))
|
||||
continue
|
||||
|
||||
old_label = f" ({existing_gps['lat']:.6f},{existing_gps['lon']:.6f})" if has_gps else " (no GPS)"
|
||||
updates.append((best["lat"], best["lon"], best["source"], mid))
|
||||
print(
|
||||
f" {mid:>6s} {old_label} → ({best['lat']:.6f}, {best['lon']:.6f}) [{best['source']:>7s}] "
|
||||
f"{existing_gps['name'][:40]}"
|
||||
)
|
||||
|
||||
# 4. Apply
|
||||
if apply_changes and updates:
|
||||
conn.executemany(
|
||||
"UPDATE assets SET latitude=?, longitude=?, location_source=?, updated_at=datetime('now') WHERE machine_id=?",
|
||||
[(lat, lon, src, mid) for lat, lon, src, mid in updates],
|
||||
)
|
||||
conn.commit()
|
||||
print(f"\n ✅ Applied {len(updates)} GPS updates")
|
||||
elif updates:
|
||||
print(f"\n Would update {len(updates)} assets. Run with --apply to write.")
|
||||
else:
|
||||
print("\n No updates needed.")
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total GPS sources combined: {len(all_mids)}")
|
||||
print(f" Updates to apply: {len(updates)}")
|
||||
print(f" Skipped (already has GPS): {len(skipped_existing)}")
|
||||
print(f" No matching asset in DB: {len(no_match)}")
|
||||
|
||||
# Verify final count
|
||||
if apply_changes:
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL")
|
||||
final_count = cur.fetchone()[0]
|
||||
print(f"\n Final assets with GPS: {final_count} / {len(existing)}")
|
||||
|
||||
# Print a few no-match examples
|
||||
if no_match:
|
||||
print(f"\n Sample unmapped machine IDs: {sorted(no_match)[:5]}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backfill: OCR all unlinked photos and update matching assets.
|
||||
|
||||
Scans uploads/photos/ for image files whose photo_path is not set on any
|
||||
asset in the database. For each unlinked photo, runs Tesseract OCR (and
|
||||
Ollama vision if available) to extract identifiers, cross-references
|
||||
against the assets DB, and updates matching assets with:
|
||||
- photo_path pointing to the photo file
|
||||
- serial_number if extracted from OCR and currently blank
|
||||
|
||||
Usage:
|
||||
python3 scripts/backfill_vision_photos.py # dry-run (report only)
|
||||
python3 scripts/backfill_vision_photos.py --apply # write changes
|
||||
python3 scripts/backfill_vision_photos.py --apply --force # overwrite existing photo_path
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from classify_makes import normalize_identifier, find_asset_by_normalized_id
|
||||
|
||||
DB_PATH = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
UPLOADS_DIR = Path(__file__).resolve().parent.parent / "uploads" / "photos"
|
||||
|
||||
PHOTO_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".bmp", ".dng", ".tiff", ".tif"}
|
||||
|
||||
|
||||
# ── OCR helpers (imported from server but keeping script self-contained) ────
|
||||
|
||||
|
||||
def _has_ollama() -> bool:
|
||||
"""Check if Ollama vision model is available."""
|
||||
try:
|
||||
import json, urllib.request
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:11434/api/generate",
|
||||
data=json.dumps({"model": "qwen2.5vl:3b", "prompt": "ping", "stream": False}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _has_tesseract() -> bool:
|
||||
"""Check if Tesseract is available."""
|
||||
try:
|
||||
import pytesseract
|
||||
pytesseract.get_tesseract_version()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ocr_via_ollama(image_path: Path) -> str:
|
||||
"""Run Ollama vision model on image, return extracted text."""
|
||||
import json, urllib.request, base64
|
||||
from PIL import Image as PILImage
|
||||
|
||||
img = PILImage.open(image_path)
|
||||
img.thumbnail((640, 480))
|
||||
data_b64 = base64.b64encode(img.tobytes()).decode()
|
||||
|
||||
with open(image_path, "rb") as f:
|
||||
b64_data = base64.b64encode(f.read()).decode()
|
||||
|
||||
payload = json.dumps({
|
||||
"model": "qwen2.5vl:3b",
|
||||
"prompt": "Extract all text, numbers, serial numbers, and IDs visible "
|
||||
"on this sticker or label. Return ONLY the raw text content, "
|
||||
"one item per line. Do not describe the image.",
|
||||
"images": [b64_data],
|
||||
"stream": False,
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:11434/api/generate",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
resp = urllib.request.urlopen(req, timeout=120)
|
||||
result = json.loads(resp.read().decode())
|
||||
return result.get("response", "").strip()
|
||||
|
||||
|
||||
def ocr_via_tesseract(image_path: Path) -> str:
|
||||
"""Run Tesseract OCR on image, return extracted text."""
|
||||
import pytesseract
|
||||
from PIL import Image as PILImage
|
||||
|
||||
img = PILImage.open(image_path)
|
||||
img_gray = img.convert("L")
|
||||
text = pytesseract.image_to_string(img_gray, config="--psm 6")
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _count_clean_chars(text: str) -> int:
|
||||
"""Count alphanumeric characters (ignore symbol noise)."""
|
||||
return sum(1 for c in text if c.isalnum() or c in ' \n/-.')
|
||||
|
||||
|
||||
def _extract_serial_from_text(text: str) -> str | None:
|
||||
"""Try to extract a plausible serial number from OCR text."""
|
||||
if not text:
|
||||
return None
|
||||
patterns = [
|
||||
r'(?:S/N|SN|SERIAL\s*NO|SERIAL|SERIAL\s*#)\s*[:=#]?\s*([A-Za-z0-9.\-/]{6,})',
|
||||
r'(?:EQUIPMENT\s*ID|EQ\s*ID|ASSET\s*ID)\s*[:=]?\s*([A-Za-z0-9.\-/]{6,})',
|
||||
r'(?:MODEL\s*NO|MODEL\s*#|PART\s*NO|P/N)\s*[:=]?\s*([A-Za-z0-9.\-/]{6,})',
|
||||
]
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text, re.IGNORECASE)
|
||||
if m:
|
||||
val = m.group(1).strip().rstrip('.')
|
||||
if re.match(r'^\d{4,}$', val.replace('-', '').replace('.', '')):
|
||||
continue # Probably a Connect ID, not a serial
|
||||
clean = sum(1 for c in val if c.isalnum())
|
||||
if clean >= 6:
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def _find_identifiers(text: str) -> list:
|
||||
"""Extract all plausible identifiers from OCR text."""
|
||||
identifiers = []
|
||||
lines = text.split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or len(line) < 4:
|
||||
continue
|
||||
norm = normalize_identifier(line)
|
||||
if norm and len(norm) >= 4:
|
||||
identifiers.append({"raw": line, "normalized": norm, "type": "full_line"})
|
||||
tokens = re.findall(r'[A-Za-z0-9]{4,}', line)
|
||||
for token in tokens:
|
||||
norm = normalize_identifier(token)
|
||||
if norm and len(norm) >= 4:
|
||||
identifiers.append({"raw": token, "normalized": norm, "type": "token"})
|
||||
return identifiers
|
||||
|
||||
|
||||
def _match_identifiers(identifiers: list, db_path: str) -> list:
|
||||
"""Match identifiers against DB, return deduplicated asset matches."""
|
||||
results = []
|
||||
seen_ids = set()
|
||||
for ident in identifiers:
|
||||
assets = find_asset_by_normalized_id(db_path, ident["normalized"])
|
||||
for a in assets:
|
||||
if a["id"] not in seen_ids:
|
||||
seen_ids.add(a["id"])
|
||||
results.append({
|
||||
"asset": a,
|
||||
"matched_on": ident["normalized"],
|
||||
"source_text": ident["raw"],
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def _get_unlinked_photos(db_path: str) -> list:
|
||||
"""Get list of photos in uploads/photos/ not linked to any asset."""
|
||||
# Get all photo_paths already in DB
|
||||
conn = sqlite3.connect(db_path)
|
||||
linked = {row[0] for row in conn.execute(
|
||||
"SELECT photo_path FROM assets WHERE photo_path IS NOT NULL AND photo_path != ''"
|
||||
).fetchall()}
|
||||
conn.close()
|
||||
|
||||
unlinked = []
|
||||
for f in sorted(UPLOADS_DIR.iterdir()):
|
||||
if f.suffix.lower() not in PHOTO_EXTS:
|
||||
continue
|
||||
rel_path = f"/uploads/photos/{f.name}"
|
||||
if rel_path not in linked:
|
||||
unlinked.append({"path": f, "rel": rel_path})
|
||||
|
||||
return unlinked
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Backfill: OCR all unlinked photos and update matching assets"
|
||||
)
|
||||
parser.add_argument("--apply", action="store_true",
|
||||
help="Write changes to DB (default: dry-run)")
|
||||
parser.add_argument("--force", action="store_true",
|
||||
help="Overwrite existing photo_path on assets")
|
||||
parser.add_argument("--db", default=DB_PATH,
|
||||
help=f"Database path (default: {DB_PATH})")
|
||||
parser.add_argument("--photos-dir", default=str(UPLOADS_DIR),
|
||||
help=f"Photos directory (default: {UPLOADS_DIR})")
|
||||
args = parser.parse_args()
|
||||
|
||||
db_path = args.db
|
||||
photos_dir = Path(args.photos_dir)
|
||||
|
||||
print(f"🔍 Scanning {photos_dir} for unlinked photos...")
|
||||
unlinked = _get_unlinked_photos(db_path)
|
||||
print(f" Found {len(unlinked)} unlinked photo(s) out of "
|
||||
f"{len(list(photos_dir.glob('*')))} total files in directory.\n")
|
||||
|
||||
if not unlinked:
|
||||
print("✅ All photos are already linked to assets. Nothing to do.")
|
||||
return
|
||||
|
||||
has_ollama = _has_ollama()
|
||||
has_tess = _has_tesseract()
|
||||
ollama_status = "✓ connected" if has_ollama else "✗ not available"
|
||||
tess_status = "✓ installed" if has_tess else "✗ not available"
|
||||
print(f" Ollama vision: {ollama_status}")
|
||||
print(f" Tesseract: {tess_status}")
|
||||
if not has_ollama and not has_tess:
|
||||
print("⚠️ No OCR service available. Install Tesseract or start Ollama.")
|
||||
return
|
||||
|
||||
total_photos = len(unlinked)
|
||||
matched_count = 0
|
||||
updated_photo_count = 0
|
||||
updated_serial_count = 0
|
||||
|
||||
for i, photo in enumerate(unlinked, 1):
|
||||
img_path = photo["path"]
|
||||
rel_path = photo["rel"]
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"[{i}/{total_photos}] {img_path.name}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Step 1: OCR
|
||||
text = ""
|
||||
ocr_source = "none"
|
||||
|
||||
if has_ollama:
|
||||
print(" [vision] Running Ollama...")
|
||||
try:
|
||||
text = ocr_via_ollama(img_path)
|
||||
if text and _count_clean_chars(text) >= 10:
|
||||
ocr_source = "ollama"
|
||||
print(f" ✓ Ollama extracted {len(text)} chars")
|
||||
except Exception as e:
|
||||
print(f" ⚠ Ollama error: {e}")
|
||||
|
||||
if ocr_source == "none" and has_tess:
|
||||
print(" [ocr] Running Tesseract...")
|
||||
try:
|
||||
text = ocr_via_tesseract(img_path)
|
||||
if text and _count_clean_chars(text) >= 10:
|
||||
ocr_source = "tesseract"
|
||||
print(f" ✓ Tesseract extracted {len(text)} chars")
|
||||
except Exception as e:
|
||||
print(f" ⚠ Tesseract error: {e}")
|
||||
|
||||
if not text or _count_clean_chars(text) < 10:
|
||||
print(" ⚠ Could not extract sufficient text from this image.")
|
||||
continue
|
||||
|
||||
print(f" Text preview: {text[:150]}")
|
||||
|
||||
# Step 2: Extract identifiers
|
||||
identifiers = _find_identifiers(text)
|
||||
if not identifiers:
|
||||
print(" ⚠ No identifiers found in OCR text.")
|
||||
continue
|
||||
|
||||
# Step 3: Match against DB
|
||||
matches = _match_identifiers(identifiers, db_path)
|
||||
if not matches:
|
||||
print(" ⚠ No DB matches found for extracted identifiers.")
|
||||
continue
|
||||
|
||||
matched_count += 1
|
||||
print(f" ✓ Matched {len(matches)} asset(s):")
|
||||
for m in matches:
|
||||
a = m["asset"]
|
||||
print(f" ├─ Asset #{a['id']}: {a['name'][:50]}")
|
||||
print(f" ├─ Machine ID: {a['machine_id']}")
|
||||
print(f" ├─ Serial: {a['serial_number'][:40] or '(blank)'}")
|
||||
print(f" └─ Matched on: {m['matched_on']}")
|
||||
|
||||
if not args.apply:
|
||||
print(" ℹ️ Dry-run — use --apply to write changes.")
|
||||
continue
|
||||
|
||||
# Step 4: Apply changes
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
for m in matches:
|
||||
a = m["asset"]
|
||||
|
||||
# Update photo_path if blank (or forced)
|
||||
needs_photo = args.force or not a.get("photo_path")
|
||||
if needs_photo:
|
||||
conn.execute(
|
||||
"UPDATE assets SET photo_path = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(rel_path, a["id"]),
|
||||
)
|
||||
updated_photo_count += 1
|
||||
print(f" ✓ Set photo_path → {rel_path}")
|
||||
|
||||
# Update serial_number if blank
|
||||
if not a.get("serial_number"):
|
||||
serial = _extract_serial_from_text(text)
|
||||
if serial:
|
||||
conn.execute(
|
||||
"UPDATE assets SET serial_number = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(serial, a["id"]),
|
||||
)
|
||||
updated_serial_count += 1
|
||||
print(f" ✓ Set serial_number → {serial}")
|
||||
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
print(f" ✗ DB error: {e}")
|
||||
conn.rollback()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*60}")
|
||||
print(f"SUMMARY")
|
||||
print(f"{'='*60}")
|
||||
print(f" Total unlinked photos: {total_photos}")
|
||||
print(f" Photos with DB matches: {matched_count}")
|
||||
if args.apply:
|
||||
print(f" Photo paths updated: {updated_photo_count}")
|
||||
print(f" Serial numbers updated: {updated_serial_count}")
|
||||
else:
|
||||
print(f" (dry-run — no changes written)")
|
||||
print(f" Re-run with --apply to write changes.")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cross-reference extracted photo IDs against assets DB.
|
||||
|
||||
Processes all photos in uploads/photos/, runs vision/OCR text extraction,
|
||||
matches identifiers against assets.machine_id / serial_number / connect_id,
|
||||
updates matched assets with photo_path, and reports unmatched IDs.
|
||||
|
||||
Usage:
|
||||
python3 scripts/crossref_photos.py # process all photos
|
||||
python3 scripts/crossref_photos.py --photo <name> # single photo
|
||||
python3 scripts/crossref_photos.py --text "S/N: 1234" # from text only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
from classify_makes import normalize_identifier, find_asset_by_normalized_id
|
||||
|
||||
DB_PATH = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
PHOTOS_DIR = str(Path(__file__).resolve().parent.parent / "uploads" / "photos")
|
||||
|
||||
|
||||
def process_photo(photo_path: str, db_path: str = DB_PATH) -> dict:
|
||||
"""Process a single photo: extract text, match DB, update photo_path."""
|
||||
fname = os.path.basename(photo_path)
|
||||
result = {
|
||||
"photo": fname,
|
||||
"photo_path": f"/uploads/photos/{fname}",
|
||||
"size_bytes": os.path.getsize(photo_path),
|
||||
"vision_text": "",
|
||||
"extracted_ids": [],
|
||||
"matches": [],
|
||||
"updated": False,
|
||||
}
|
||||
|
||||
# Try Tesseract OCR
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image as PILImage
|
||||
|
||||
img = PILImage.open(photo_path)
|
||||
img_gray = img.convert("L")
|
||||
text = pytesseract.image_to_string(img_gray, config="--psm 6")
|
||||
text = text.strip()
|
||||
clean_chars = sum(1 for c in text if c.isalnum() or c in " \n/-.")
|
||||
if text and clean_chars >= 4:
|
||||
result["vision_text"] = text
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if not result["vision_text"]:
|
||||
result["vision_source"] = "none"
|
||||
return result
|
||||
|
||||
result["vision_source"] = "tesseract"
|
||||
|
||||
# Extract identifiers from text
|
||||
identifiers = []
|
||||
lines = result["vision_text"].split("\n")
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or len(line) < 4:
|
||||
continue
|
||||
norm = normalize_identifier(line)
|
||||
if norm and len(norm) >= 4:
|
||||
identifiers.append(norm)
|
||||
tokens = re.findall(r"[A-Za-z0-9]{4,}", line)
|
||||
for token in tokens:
|
||||
norm = normalize_identifier(token)
|
||||
if norm and len(norm) >= 4 and norm not in identifiers:
|
||||
identifiers.append(norm)
|
||||
|
||||
result["extracted_ids"] = identifiers
|
||||
|
||||
# Match against DB
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
seen_ids = set()
|
||||
|
||||
for norm_id in identifiers:
|
||||
assets = find_asset_by_normalized_id(db_path, norm_id)
|
||||
for a in assets:
|
||||
if a["id"] not in seen_ids:
|
||||
seen_ids.add(a["id"])
|
||||
result["matches"].append(a)
|
||||
|
||||
# Update photo_path if empty
|
||||
existing = conn.execute(
|
||||
"SELECT photo_path FROM assets WHERE id = ?", (a["id"],)
|
||||
).fetchone()
|
||||
if existing and not existing["photo_path"]:
|
||||
conn.execute(
|
||||
"UPDATE assets SET photo_path = ?, updated_at = datetime('now') WHERE id = ?",
|
||||
(result["photo_path"], a["id"]),
|
||||
)
|
||||
result["updated"] = True
|
||||
conn.commit()
|
||||
|
||||
conn.close()
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Cross-reference photo IDs against assets DB"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--photo", "-p", help="Process a single photo by filename"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text", "-t", help="Process raw text directly (skip OCR)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db",
|
||||
default=DB_PATH,
|
||||
help=f"Database path (default: {DB_PATH})",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
results = []
|
||||
|
||||
if args.text:
|
||||
# Process from text directly
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"PROCESSING: supplied text ({len(args.text)} chars)")
|
||||
print(f"{'=' * 70}")
|
||||
print(f" Text: {args.text[:500]}")
|
||||
|
||||
identifiers = []
|
||||
norm = normalize_identifier(args.text)
|
||||
if norm and len(norm) >= 4:
|
||||
identifiers.append(norm)
|
||||
tokens = re.findall(r"[A-Za-z0-9]{4,}", args.text)
|
||||
for token in tokens:
|
||||
n = normalize_identifier(token)
|
||||
if n and len(n) >= 4 and n not in identifiers:
|
||||
identifiers.append(n)
|
||||
|
||||
print(f"\n Normalized IDs: {identifiers}")
|
||||
conn = sqlite3.connect(args.db)
|
||||
conn.row_factory = sqlite3.Row
|
||||
for norm_id in identifiers:
|
||||
assets = find_asset_by_normalized_id(args.db, norm_id)
|
||||
if assets:
|
||||
for a in assets:
|
||||
print(f"\n ✓ MATCH → Asset #{a['id']}")
|
||||
print(f" Machine ID: {a['machine_id']}")
|
||||
print(f" Name: {a['name'][:60]}")
|
||||
else:
|
||||
print(f"\n ✗ No match for '{norm_id}'")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
if args.photo:
|
||||
photo_path = os.path.join(PHOTOS_DIR, args.photo)
|
||||
if not os.path.exists(photo_path):
|
||||
print(f"ERROR: {photo_path} not found")
|
||||
sys.exit(1)
|
||||
results.append(process_photo(photo_path, args.db))
|
||||
else:
|
||||
# Process all photos
|
||||
for fname in sorted(os.listdir(PHOTOS_DIR)):
|
||||
photo_path = os.path.join(PHOTOS_DIR, fname)
|
||||
if os.path.isfile(photo_path):
|
||||
results.append(process_photo(photo_path, args.db))
|
||||
|
||||
# Print summary
|
||||
matched = [r for r in results if r["matches"]]
|
||||
unmatched = [r for r in results if not r["matches"] and r["vision_text"]]
|
||||
no_text = [r for r in results if not r["vision_text"]]
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"CROSS-REFERENCE SUMMARY ({datetime.now().isoformat()})")
|
||||
print(f"{'=' * 70}")
|
||||
print(f" Photos scanned: {len(results)}")
|
||||
print(f" Photos with text: {len(results) - len(no_text)}")
|
||||
print(f" Matches found: {len(matched)}")
|
||||
print(f" Unmatched: {len(unmatched)}")
|
||||
print(f" No text (small/thumbs): {len(no_text)}")
|
||||
|
||||
for r in matched:
|
||||
for a in r["matches"]:
|
||||
status = "UPDATED" if r["updated"] else "already set"
|
||||
print(
|
||||
f"\n ✓ {r['photo']} → Asset #{a['id']} "
|
||||
f"(machine_id={a['machine_id']}) [{status}]"
|
||||
)
|
||||
|
||||
for r in unmatched:
|
||||
print(f"\n ✗ {r['photo']} — no DB match")
|
||||
print(f" IDs: {r['extracted_ids'][:5]}")
|
||||
print(f" Text preview: {r['vision_text'][:100]}")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Geocode all asset addresses from the assets DB, cache results, and update GPS.
|
||||
|
||||
Sources (priority):
|
||||
1. Existing geocache.json
|
||||
2. Fresh Nominatim (OpenStreetMap) geocoding — rate-limited to 1/sec
|
||||
|
||||
Usage:
|
||||
python3 scripts/geocode_asset_addresses.py # dry-run
|
||||
python3 scripts/geocode_asset_addresses.py --apply # write to DB
|
||||
python3 scripts/geocode_asset_addresses.py --force # re-geocode cached too
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ASSET_DB = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
GEOCODE_CACHE = str(
|
||||
Path.home()
|
||||
/ "projects"
|
||||
/ "ms-field-service-extraction"
|
||||
/ "web"
|
||||
/ "static"
|
||||
/ "data"
|
||||
/ "geocache.json"
|
||||
)
|
||||
|
||||
USER_AGENT = "CanteenAssetTracker/1.0 (internal tool; canteen.ourpad.casa)"
|
||||
NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
|
||||
RATE_LIMIT = 1.1 # seconds between requests
|
||||
|
||||
|
||||
def load_geocache():
|
||||
"""Load existing geocache from file."""
|
||||
if Path(GEOCODE_CACHE).exists():
|
||||
with open(GEOCODE_CACHE) as f:
|
||||
return json.load(f)
|
||||
return {}
|
||||
|
||||
|
||||
def save_geocache(cache):
|
||||
"""Save geocache back to file."""
|
||||
Path(GEOCODE_CACHE).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(GEOCODE_CACHE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
|
||||
|
||||
def geocode(address):
|
||||
"""Geocode a single address via Nominatim. Returns (lat, lng) or (None, None)."""
|
||||
if not address or not address.strip():
|
||||
return None, None
|
||||
|
||||
params = urllib.parse.urlencode({
|
||||
"q": address.strip(),
|
||||
"format": "json",
|
||||
"limit": 1,
|
||||
"addressdetails": "0",
|
||||
})
|
||||
url = f"{NOMINATIM_URL}?{params}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read())
|
||||
if data and len(data) > 0:
|
||||
lat = float(data[0]["lat"])
|
||||
lng = float(data[0]["lon"])
|
||||
return lat, lng
|
||||
except Exception as e:
|
||||
print(f" ⚠ Error geocoding '{address[:60]}': {e}", file=sys.stderr)
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def make_addr_key(asset):
|
||||
"""Build a normalized address key from asset record."""
|
||||
parts = [
|
||||
asset.get("address", "") or "",
|
||||
asset.get("seed_location", "") or "",
|
||||
]
|
||||
# Build full address with city/state
|
||||
addr = ""
|
||||
if asset.get("address"):
|
||||
addr = asset["address"].strip()
|
||||
if asset.get("seed_city"):
|
||||
addr += f", {asset['seed_city'].strip()}"
|
||||
if asset.get("state"):
|
||||
addr += f", {asset['state'].strip()}"
|
||||
if asset.get("postal_code"):
|
||||
addr += f", {asset['postal_code'].strip()}"
|
||||
elif asset.get("seed_location"):
|
||||
addr = asset["seed_location"].strip()
|
||||
|
||||
return addr.strip().lower() if addr else None
|
||||
|
||||
|
||||
def build_full_address(asset):
|
||||
"""Build the cleanest full address string for geocoding.
|
||||
|
||||
Uses only the street address + city + state + zip.
|
||||
Excludes seed_location (extra venue/building names confuse Nominatim).
|
||||
"""
|
||||
addr = (asset.get("address") or "").strip()
|
||||
if not addr:
|
||||
addr = (asset.get("seed_location") or "").strip()
|
||||
|
||||
loc_parts = []
|
||||
if asset.get("seed_city"):
|
||||
loc_parts.append(asset["seed_city"].strip())
|
||||
if asset.get("state"):
|
||||
loc_parts.append(asset["state"].strip())
|
||||
if asset.get("postal_code"):
|
||||
loc_parts.append(asset["postal_code"].strip())
|
||||
|
||||
if loc_parts:
|
||||
# Add "FL" if not already present in the state field
|
||||
if "FL" not in loc_parts and "Florida" not in loc_parts:
|
||||
loc_parts.insert(0, "FL")
|
||||
addr += ", " + ", ".join(loc_parts)
|
||||
elif not addr:
|
||||
return None
|
||||
|
||||
return addr.strip().strip(",").strip()
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
force = "--force" in sys.argv
|
||||
|
||||
mode = "DRY RUN (no writes)" if not apply_changes else "APPLYING changes"
|
||||
print(f"Assets DB: {ASSET_DB}")
|
||||
print(f"Geocache: {GEOCODE_CACHE}")
|
||||
print(f"Mode: {mode}")
|
||||
if force:
|
||||
print(" (--force: re-geocode even cached addresses)")
|
||||
print()
|
||||
|
||||
# 1. Load existing cache
|
||||
cache = load_geocache()
|
||||
cached_with_coords = sum(1 for v in cache.values() if v.get("lat"))
|
||||
cached_misses = sum(1 for v in cache.values() if not v.get("lat"))
|
||||
print(f"📦 Geocache: {len(cache)} entries ({cached_with_coords} with coords, {cached_misses} misses)")
|
||||
|
||||
# 2. Get all assets without GPS but with addresses
|
||||
conn = sqlite3.connect(ASSET_DB)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("""
|
||||
SELECT machine_id, name, address, seed_location, seed_city, state, postal_code
|
||||
FROM assets
|
||||
WHERE latitude IS NULL
|
||||
AND ((address IS NOT NULL AND address != '')
|
||||
OR (seed_location IS NOT NULL AND seed_location != ''))
|
||||
ORDER BY machine_id
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
print(f"📋 Assets needing geocoding: {len(rows)}")
|
||||
|
||||
if not rows:
|
||||
print("✅ No assets need geocoding. Done.")
|
||||
return
|
||||
|
||||
# 3. Build (addr_key -> machine_ids) map for unique addresses
|
||||
addr_to_mids = {}
|
||||
addr_to_full = {}
|
||||
asset_cols = ["machine_id", "name", "address", "seed_location", "seed_city", "state", "postal_code"]
|
||||
col = dict(zip(asset_cols, range(len(asset_cols))))
|
||||
|
||||
for row in rows:
|
||||
asset = {k: row[col[k]] for k in asset_cols}
|
||||
addr_key = make_addr_key(asset)
|
||||
if not addr_key:
|
||||
continue
|
||||
if addr_key not in addr_to_mids:
|
||||
addr_to_mids[addr_key] = []
|
||||
addr_to_full[addr_key] = build_full_address(asset)
|
||||
addr_to_mids[addr_key].append(row[col["machine_id"]])
|
||||
|
||||
print(f"📍 Unique addresses to resolve: {len(addr_to_mids)}")
|
||||
|
||||
# 4. Resolve each unique address
|
||||
resolved = {} # addr_key -> (lat, lon)
|
||||
still_needed = 0
|
||||
fresh_calls = 0
|
||||
|
||||
for addr_key, mids in sorted(addr_to_mids.items()):
|
||||
# Check cache first
|
||||
cached = cache.get(addr_key, {})
|
||||
if cached and not force:
|
||||
lat, lng = cached.get("lat"), cached.get("lng")
|
||||
if lat and lng:
|
||||
resolved[addr_key] = (lat, lng)
|
||||
continue
|
||||
elif cached.get("lat") is None: # previously cached as miss
|
||||
still_needed += 1
|
||||
continue
|
||||
|
||||
# Need to geocode fresh
|
||||
full_addr = addr_to_full[addr_key]
|
||||
print(f" 🌐 Geocoding [{fresh_calls+1}]: {full_addr[:70]}...")
|
||||
lat, lng = geocode(full_addr)
|
||||
|
||||
# Save to cache
|
||||
cache[addr_key] = {"address": full_addr, "lat": lat, "lng": lng}
|
||||
save_geocache(cache)
|
||||
|
||||
if lat and lng:
|
||||
resolved[addr_key] = (lat, lng)
|
||||
print(f" ✅ ({lat:.5f}, {lng:.5f}) — affects {len(mids)} assets")
|
||||
else:
|
||||
print(f" ❌ No result — affects {len(mids)} assets")
|
||||
still_needed += 1
|
||||
|
||||
fresh_calls += 1
|
||||
if fresh_calls >= 3 and not apply_changes:
|
||||
# In dry-run mode, only do a few to show what's possible
|
||||
print(f"\n ... stopping after {fresh_calls} (dry-run). Use --apply to geocode all ~{len(addr_to_mids)} addresses.")
|
||||
break
|
||||
time.sleep(RATE_LIMIT)
|
||||
|
||||
# 5. Count updates
|
||||
total_updatable = 0
|
||||
updates = []
|
||||
for addr_key, mids in addr_to_mids.items():
|
||||
if addr_key in resolved:
|
||||
lat, lon = resolved[addr_key]
|
||||
for mid in mids:
|
||||
updates.append((lat, lon, "geocoded", mid))
|
||||
total_updatable += len(mids)
|
||||
|
||||
updates_by_source = sum(len(m) for a, m in addr_to_mids.items() if a in resolved)
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"📊 SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total assets needing GPS: {len(rows)}")
|
||||
print(f" Unique addresses to geocode: {len(addr_to_mids)}")
|
||||
print(f" Resolved from cache/source: {len(resolved) + (cached_with_coords - sum(1 for k in resolved if k not in cache))}")
|
||||
print(f" Freshly geocoded: {fresh_calls}")
|
||||
print(f" Still unresolved: {still_needed}")
|
||||
print(f" Assets that can be updated: {updates_by_source}")
|
||||
|
||||
if apply_changes and updates:
|
||||
# Apply in batches for speed
|
||||
conn.executemany(
|
||||
"UPDATE assets SET latitude=?, longitude=?, location_source=?, updated_at=datetime('now') WHERE machine_id=?",
|
||||
[(lat, lon, src, mid) for lat, lon, src, mid in updates],
|
||||
)
|
||||
conn.commit()
|
||||
print(f"\n ✅ Applied {len(updates)} GPS updates from geocoding")
|
||||
elif updates:
|
||||
print(f"\n Run with --apply to write {len(updates)} GPS updates to DB.")
|
||||
|
||||
# Final count
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL")
|
||||
final_gps = cur.fetchone()[0]
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets")
|
||||
total = cur.fetchone()[0]
|
||||
print(f"\n Final: {final_gps} / {total} assets have GPS coordinates")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,411 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import missing machines from Cantaloupe Excel export into canteen-asset-tracker.
|
||||
|
||||
Reads Machine List(8).xlsx, checks each Asset ID against assets.db,
|
||||
and inserts any missing records. Stores full row JSON in seed_data table.
|
||||
|
||||
Usage:
|
||||
python3 import_machines.py # Dry run (no writes)
|
||||
python3 import_machines.py --write # Actually import
|
||||
python3 import_machines.py --write --prod # Import to production DB
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import openpyxl
|
||||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────────
|
||||
EXCEL_PATH = os.path.expanduser(
|
||||
"~/.hermes/profiles/coder/cache/documents/doc_4589f73a5e74_Machine List(8).xlsx"
|
||||
)
|
||||
|
||||
DEV_DB = os.path.expanduser("~/projects/canteen-asset-tracker-dev/assets.dev.db")
|
||||
PROD_DB = os.path.expanduser("~/projects/canteen-asset-tracker/assets.db")
|
||||
|
||||
# ── Column index mapping (0-based) ─────────────────────────────────────────
|
||||
COL = {
|
||||
"device": 0,
|
||||
"location": 1,
|
||||
"asset_id": 2,
|
||||
"place": 3,
|
||||
"type": 4,
|
||||
"city": 5,
|
||||
"address": 6,
|
||||
"last_contact_time": 7,
|
||||
"last_dex_report_time": 8,
|
||||
"last_restock": 9,
|
||||
"coil_alerts": 10,
|
||||
"product_alerts": 11,
|
||||
"sales_restock": 12,
|
||||
"daily_avg_sales": 13,
|
||||
"today_sales": 14,
|
||||
"yesterday_sales": 15,
|
||||
"weekly_sales": 16,
|
||||
"monthly_sales": 17,
|
||||
"yearly_sales": 18,
|
||||
"days_since_restock": 19,
|
||||
"prepick_group": 20,
|
||||
"customer": 21,
|
||||
"management_company": 22,
|
||||
"management_account": 23,
|
||||
"machine_management_code": 24,
|
||||
"route": 25,
|
||||
"subroute": 26,
|
||||
"changer_par": 27,
|
||||
"acquired_from": 28,
|
||||
"purchase_date": 29,
|
||||
"purchase_price": 30,
|
||||
"depreciation_years": 31,
|
||||
"post_depr_monthly_cost": 32,
|
||||
"state": 33,
|
||||
"postal_code": 34,
|
||||
"deployed": 35,
|
||||
"pulled_date": 36,
|
||||
"serial_number": 37,
|
||||
"class": 38,
|
||||
"make": 39,
|
||||
"model": 40,
|
||||
"cash_discount": 41,
|
||||
"tax_jurisdiction": 42,
|
||||
"commission_plan": 43,
|
||||
"barcode": 44,
|
||||
"non_revenue": 45,
|
||||
"added_date": 46,
|
||||
"phone": 47,
|
||||
"fax": 48,
|
||||
"email": 49,
|
||||
"has_cashless": 50,
|
||||
"branch": 51,
|
||||
"location_code": 52,
|
||||
"customer_code": 53,
|
||||
"last_inventory": 54,
|
||||
"asset_family": 55,
|
||||
"status": 56,
|
||||
"alerts": 57,
|
||||
"valid_address": 58,
|
||||
"business_type": 59,
|
||||
"primary_consumer_type": 60,
|
||||
"machine_branding": 61,
|
||||
}
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def cell(row, key, default=""):
|
||||
"""Get cell value by column key."""
|
||||
idx = COL[key]
|
||||
val = row[idx]
|
||||
if val is None:
|
||||
return default
|
||||
if isinstance(val, datetime):
|
||||
return val.isoformat()
|
||||
return str(val).strip()
|
||||
|
||||
|
||||
def cell_num(row, key, default=None):
|
||||
"""Get numeric cell value."""
|
||||
idx = COL[key]
|
||||
val = row[idx]
|
||||
if val is None:
|
||||
return default
|
||||
if isinstance(val, (int, float)):
|
||||
return val
|
||||
try:
|
||||
return float(str(val).strip())
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def cell_int(row, key, default=None):
|
||||
"""Get integer cell value."""
|
||||
n = cell_num(row, key, default)
|
||||
if n is not None:
|
||||
return int(n)
|
||||
return None
|
||||
|
||||
|
||||
def parse_type(raw_type):
|
||||
"""Parse 'Snack (AMS Sensit 3)' -> (category='Snack', make='AMS', model='Sensit 3')"""
|
||||
if not raw_type:
|
||||
return "Other", "", ""
|
||||
raw = raw_type.strip()
|
||||
# Extract parenthetical part
|
||||
paren_match = re.search(r"\((.+?)\)", raw)
|
||||
paren = paren_match.group(1) if paren_match else ""
|
||||
|
||||
# Get the category (first word or before parentheses)
|
||||
category = re.sub(r"\s*\(.*\).*", "", raw).strip()
|
||||
if not category or category == "Unknown":
|
||||
category = "Other"
|
||||
|
||||
# Parse make/model from parentheses
|
||||
if paren:
|
||||
parts = paren.split(None, 1)
|
||||
make = parts[0] if len(parts) > 0 else ""
|
||||
model = parts[1] if len(parts) > 1 else ""
|
||||
else:
|
||||
make = ""
|
||||
model = ""
|
||||
|
||||
return category, make, model
|
||||
|
||||
|
||||
def build_name(row):
|
||||
"""Build asset name in format: 'Make @ Location' or 'AssetID / Address / Place'"""
|
||||
loc = cell(row, "location")
|
||||
place = cell(row, "place")
|
||||
aid = cell(row, "asset_id")
|
||||
_, make, model = parse_type(cell(row, "type"))
|
||||
addr = cell(row, "address")
|
||||
|
||||
if make:
|
||||
name = f"{make} @ {place or loc or addr or aid}"
|
||||
else:
|
||||
name = f"{aid} / {addr or loc} / {place}" if (addr or place) else aid
|
||||
|
||||
return name[:250] # DB column limit
|
||||
|
||||
|
||||
def deployed_flag(val):
|
||||
"""Convert deployed value to '1' or '0'."""
|
||||
if isinstance(val, str) and val.lower() in ("yes", "y", "1", "true"):
|
||||
return "1"
|
||||
if isinstance(val, (int, float)):
|
||||
return "1" if val else "0"
|
||||
return "0"
|
||||
|
||||
|
||||
# ── Import logic ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def import_machines(target_db, dry_run=True):
|
||||
"""Import missing machines from Excel into the target DB."""
|
||||
|
||||
if not os.path.exists(EXCEL_PATH):
|
||||
print(f"ERROR: Excel file not found: {EXCEL_PATH}")
|
||||
return
|
||||
|
||||
if not os.path.exists(target_db):
|
||||
print(f"ERROR: Target DB not found: {target_db}")
|
||||
return
|
||||
|
||||
# Load spreadsheet
|
||||
wb = openpyxl.load_workbook(EXCEL_PATH)
|
||||
ws = wb["Machine List"]
|
||||
|
||||
# Read all rows
|
||||
all_rows = []
|
||||
for r in ws.iter_rows(min_row=2, values_only=True):
|
||||
aid = r[COL["asset_id"]]
|
||||
if aid:
|
||||
all_rows.append(r)
|
||||
|
||||
print(f"Total machines in spreadsheet: {len(all_rows)}")
|
||||
|
||||
# Connect to target DB
|
||||
conn = sqlite3.connect(target_db)
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Get existing machine_ids
|
||||
existing = set()
|
||||
for r in conn.execute("SELECT machine_id FROM assets WHERE machine_id IS NOT NULL"):
|
||||
existing.add(str(r["machine_id"]).strip())
|
||||
|
||||
print(f"Existing machines in DB: {len(existing)}")
|
||||
|
||||
# Find missing
|
||||
missing_rows = []
|
||||
for row in all_rows:
|
||||
aid = str(row[COL["asset_id"]]).strip()
|
||||
if aid not in existing:
|
||||
missing_rows.append((aid, row))
|
||||
|
||||
print(f"Missing (to import): {len(missing_rows)}")
|
||||
missing_rows.sort(key=lambda x: int(x[0]))
|
||||
|
||||
if not missing_rows:
|
||||
print("Nothing to import.")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# Show first 10
|
||||
print(f"\nFirst 10 to import:")
|
||||
for aid, row in missing_rows[:10]:
|
||||
loc = cell(row, "location")
|
||||
place = cell(row, "place")
|
||||
typ = cell(row, "type")
|
||||
print(f" {aid}: {loc[:60]} | {place} | {typ}")
|
||||
|
||||
if dry_run:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"DRY RUN - no changes made. Re-run with --write to import.")
|
||||
print(f"{'='*60}")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# ── Do the import ─────────────────────────────────────────────────
|
||||
conn.execute("PRAGMA foreign_keys = OFF")
|
||||
conn.execute("BEGIN TRANSACTION")
|
||||
|
||||
imported_count = 0
|
||||
errors = []
|
||||
seed_entries = []
|
||||
|
||||
for aid, row in missing_rows:
|
||||
try:
|
||||
cat, make, model = parse_type(cell(row, "type"))
|
||||
# Use spreadsheet make/model if type parse didn't yield them
|
||||
sp_make = cell(row, "make")
|
||||
if sp_make:
|
||||
make = sp_make
|
||||
sp_model = cell(row, "model")
|
||||
if sp_model:
|
||||
model = sp_model
|
||||
|
||||
asset_data = {
|
||||
"machine_id": aid,
|
||||
"serial_number": cell(row, "serial_number"),
|
||||
"name": build_name(row),
|
||||
"category": cat,
|
||||
"status": cell(row, "status") or "active",
|
||||
"make": make,
|
||||
"model": model,
|
||||
"address": cell(row, "address"),
|
||||
"customer_name": cell(row, "customer"),
|
||||
"company": cell(row, "customer"),
|
||||
"place": cell(row, "place"),
|
||||
"seed_city": cell(row, "city"),
|
||||
"state": cell(row, "state"),
|
||||
"postal_code": cell(row, "postal_code"),
|
||||
"route_name": cell(row, "route"),
|
||||
"subroute_name": cell(row, "subroute"),
|
||||
"branch": cell(row, "branch"),
|
||||
"barcode": cell(row, "barcode"),
|
||||
"seed_class": cell(row, "class"),
|
||||
"device": cell(row, "device"),
|
||||
"management_company": cell(row, "management_company"),
|
||||
"machine_management_code": cell(row, "machine_management_code"),
|
||||
"location_code": cell(row, "location_code"),
|
||||
"customer_code": cell(row, "customer_code"),
|
||||
"asset_family": cell(row, "asset_family"),
|
||||
"business_type": cell(row, "business_type"),
|
||||
"primary_consumer_type": cell(row, "primary_consumer_type"),
|
||||
"machine_branding": cell(row, "machine_branding"),
|
||||
"valid_address": cell(row, "valid_address"),
|
||||
"phone": cell(row, "phone"),
|
||||
"fax": cell(row, "fax"),
|
||||
"email": cell(row, "email"),
|
||||
"has_cashless": "1" if cell(row, "has_cashless", "").lower() in ("yes", "y", "1") else "0",
|
||||
"non_revenue": cell(row, "non_revenue"),
|
||||
"alerts": cell(row, "alerts"),
|
||||
"coil_alerts": cell_int(row, "coil_alerts", 0),
|
||||
"product_alerts": cell_int(row, "product_alerts", 0),
|
||||
"daily_avg_sales": cell_num(row, "daily_avg_sales"),
|
||||
"monthly_sales": cell_num(row, "monthly_sales"),
|
||||
"yearly_sales": cell_num(row, "yearly_sales"),
|
||||
"today_sales": cell_num(row, "today_sales"),
|
||||
"yesterday_sales": cell_num(row, "yesterday_sales"),
|
||||
"weekly_sales": cell_num(row, "weekly_sales"),
|
||||
"sales_restock": cell_num(row, "sales_restock"),
|
||||
"last_restock": cell(row, "last_restock"),
|
||||
"days_since_restock": cell_int(row, "days_since_restock"),
|
||||
"last_contact_time": cell(row, "last_contact_time"),
|
||||
"last_dex_report_time": cell(row, "last_dex_report_time"),
|
||||
"last_inventory": cell(row, "last_inventory"),
|
||||
"prepick_group": cell(row, "prepick_group"),
|
||||
"added_date": cell(row, "added_date"),
|
||||
"install_date": cell(row, "added_date"), # seed has no install_date separate
|
||||
"purchase_date": cell(row, "purchase_date"),
|
||||
"purchase_price": cell_num(row, "purchase_price"),
|
||||
"depreciation_years": cell_int(row, "depreciation_years"),
|
||||
"cash_discount": cell_num(row, "cash_discount", 0),
|
||||
"tax_jurisdiction": cell(row, "tax_jurisdiction"),
|
||||
"commission_plan": cell(row, "commission_plan"),
|
||||
"acquirer_from": cell(row, "acquired_from"),
|
||||
"changer_par": cell(row, "changer_par"),
|
||||
"deployed": deployed_flag(row[COL["deployed"]]) if row[COL["deployed"]] else "1",
|
||||
"pulled_date": cell(row, "pulled_date"),
|
||||
"geofence_radius_meters": 50,
|
||||
}
|
||||
|
||||
# Build column names and placeholders
|
||||
cols = list(asset_data.keys())
|
||||
placeholders = ", ".join("?" for _ in cols)
|
||||
col_names = ", ".join(cols)
|
||||
vals = [asset_data[c] for c in cols]
|
||||
|
||||
conn.execute(
|
||||
f"INSERT INTO assets ({col_names}) VALUES ({placeholders})",
|
||||
vals,
|
||||
)
|
||||
|
||||
# Store raw seed data
|
||||
# Get the asset id we just inserted
|
||||
new_id = conn.execute(
|
||||
"SELECT id FROM assets WHERE machine_id = ?", (aid,)
|
||||
).fetchone()["id"]
|
||||
|
||||
# Build raw_data dict from all spreadsheet cells
|
||||
raw_data = {}
|
||||
headers = [c.value for c in ws[1]]
|
||||
for i, h in enumerate(headers):
|
||||
v = row[i]
|
||||
if isinstance(v, datetime):
|
||||
v = v.isoformat()
|
||||
raw_data[h] = v
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO seed_data (asset_id, raw_data) VALUES (?, ?)",
|
||||
(new_id, json.dumps(raw_data, default=str)),
|
||||
)
|
||||
|
||||
imported_count += 1
|
||||
|
||||
except Exception as e:
|
||||
errors.append(f" {aid}: {e}")
|
||||
|
||||
if errors:
|
||||
conn.rollback()
|
||||
print(f"\nERRORS - rolled back entire import:")
|
||||
for e in errors:
|
||||
print(e)
|
||||
else:
|
||||
conn.commit()
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
|
||||
conn.close()
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Import complete!")
|
||||
print(f" Imported: {imported_count} machines")
|
||||
print(f" Errors: {len(errors)}")
|
||||
if imported_count:
|
||||
print(f" First: {missing_rows[0][0]}")
|
||||
print(f" Last: {missing_rows[-1][0]}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Import machines from Excel into canteen asset tracker")
|
||||
parser.add_argument("--write", action="store_true", help="Actually write to DB (default: dry run)")
|
||||
parser.add_argument("--prod", action="store_true", help="Import to production DB (default: dev)")
|
||||
args = parser.parse_args()
|
||||
|
||||
target = PROD_DB if args.prod else DEV_DB
|
||||
mode = "LIVE IMPORT" if args.write else "DRY RUN"
|
||||
label = "PRODUCTION" if args.prod else "DEV"
|
||||
print(f"{'='*60}")
|
||||
print(f" {mode} -> {label} DB")
|
||||
print(f" Target: {target}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
import_machines(target, dry_run=not args.write)
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Match a sticker/label photo against the assets database.
|
||||
|
||||
Usage:
|
||||
python3 scripts/match_label_photo.py <image_path>
|
||||
|
||||
Runs OCR (Tesseract) on the image, extracts identifiers,
|
||||
and searches the assets DB for matches by serial_number, connect_id,
|
||||
equipment_id, and barcode.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from classify_makes import normalize_identifier, find_asset_by_normalized_id
|
||||
|
||||
DB_PATH = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
|
||||
|
||||
def ocr_image(image_path: str) -> str:
|
||||
"""Run Tesseract OCR on an image and return extracted text."""
|
||||
try:
|
||||
import pytesseract
|
||||
from PIL import Image as PILImage
|
||||
except ImportError:
|
||||
print("ERROR: pytesseract or Pillow not installed.")
|
||||
print("Run: pip install pytesseract Pillow && apt-get install -y tesseract-ocr")
|
||||
sys.exit(1)
|
||||
|
||||
img = PILImage.open(image_path)
|
||||
img_gray = img.convert("L")
|
||||
text = pytesseract.image_to_string(img_gray, config="--psm 6")
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _count_clean_chars(text: str) -> int:
|
||||
"""Count alphanumeric characters (ignore symbol noise)."""
|
||||
return sum(1 for c in text if c.isalnum() or c in ' \n/-.')
|
||||
|
||||
|
||||
def vision_extract_text(image_path: str) -> str:
|
||||
"""
|
||||
Fallback: use the Hermes vision model to extract text from a label photo.
|
||||
|
||||
Returns the raw text the vision model sees on the label.
|
||||
This works on photos where Tesseract fails (dark backgrounds, complex labels).
|
||||
"""
|
||||
import json, urllib.request, os
|
||||
|
||||
# Try to read vision config from Hermes profile
|
||||
config_path = os.path.expanduser("~/.hermes/profiles/coder/config.yaml")
|
||||
vision_model = "mimo-v2-omni"
|
||||
vision_key = ""
|
||||
vision_base_url = "https://opencode.ai/zen/go/v1"
|
||||
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path) as f:
|
||||
for line in f:
|
||||
if 'model:' in line and 'vision' in line or True:
|
||||
pass
|
||||
line = line.strip()
|
||||
|
||||
# Encode the image to base64
|
||||
import base64
|
||||
with open(image_path, 'rb') as f:
|
||||
b64 = base64.b64encode(f.read()).decode()
|
||||
|
||||
# Determine media type
|
||||
ext = Path(image_path).suffix.lower()
|
||||
media_type = {
|
||||
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png', '.webp': 'image/webp',
|
||||
}.get(ext, 'image/jpeg')
|
||||
|
||||
data = json.dumps({
|
||||
"model": vision_model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Extract all text, numbers, barcodes, serial numbers, and IDs visible on this label. Return ONLY the raw text content, one item per line. Do not describe the image."},
|
||||
{"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{b64}"}}
|
||||
]
|
||||
}
|
||||
],
|
||||
"max_tokens": 500
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{vision_base_url}/chat/completions",
|
||||
data=data,
|
||||
headers={
|
||||
"Authorization": f"Bearer {vision_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=30)
|
||||
result = json.loads(resp.read().decode())
|
||||
return result["choices"][0]["message"]["content"].strip()
|
||||
except Exception as e:
|
||||
return f"[Vision error: {e}]"
|
||||
|
||||
|
||||
def find_identifiers(text: str) -> list:
|
||||
"""Extract all plausible identifiers from OCR text."""
|
||||
identifiers = []
|
||||
lines = text.split('\n')
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or len(line) < 4:
|
||||
continue
|
||||
norm = normalize_identifier(line)
|
||||
if norm and len(norm) >= 4:
|
||||
identifiers.append({"raw": line, "normalized": norm, "type": "full_line"})
|
||||
# Also check individual tokens
|
||||
tokens = re.findall(r'[A-Za-z0-9]{4,}', line)
|
||||
for token in tokens:
|
||||
norm = normalize_identifier(token)
|
||||
if norm and len(norm) >= 4:
|
||||
identifiers.append({"raw": token, "normalized": norm, "type": "token"})
|
||||
return identifiers
|
||||
|
||||
|
||||
def match_identifiers(identifiers: list, db_path: str) -> list:
|
||||
"""Match identifiers against DB, return (identifier, matched_assets) pairs."""
|
||||
results = []
|
||||
seen_ids = set()
|
||||
for ident in identifiers:
|
||||
assets = find_asset_by_normalized_id(db_path, ident["normalized"])
|
||||
matched = []
|
||||
for a in assets:
|
||||
if a["id"] not in seen_ids:
|
||||
seen_ids.add(a["id"])
|
||||
matched.append(a)
|
||||
if matched:
|
||||
results.append({"identifier": ident, "matches": matched})
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Match label photo text against assets DB")
|
||||
parser.add_argument("images", nargs="*", metavar="IMAGE", help="Image file(s) to OCR")
|
||||
parser.add_argument("--text", "-t", help="Raw text to match (skip OCR)")
|
||||
parser.add_argument("--db", default=DB_PATH, help=f"Database path (default: {DB_PATH})")
|
||||
args = parser.parse_args()
|
||||
|
||||
db_path = args.db
|
||||
|
||||
if args.text:
|
||||
text = args.text
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Processing: supplied text ({len(text)} chars)")
|
||||
print(f"{'='*60}")
|
||||
print(f" Text: {text[:500]}")
|
||||
_process_text(text, db_path)
|
||||
return
|
||||
|
||||
for img_path in args.images:
|
||||
if not Path(img_path).exists():
|
||||
print(f"\n=== SKIP: {img_path} (not found) ===")
|
||||
continue
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Processing: {img_path}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Step 1: OCR
|
||||
print("\n[OCR] Running Tesseract...")
|
||||
text = ocr_image(img_path)
|
||||
print(f" Raw text:\n{text[:500]}")
|
||||
print(f" (length: {len(text)} chars, clean: {_count_clean_chars(text)})")
|
||||
|
||||
if not text or _count_clean_chars(text) < 10:
|
||||
print("\n[OCR] Tesseract produced poor/no results. The app will use its")
|
||||
print(" vision API as a fallback when processing through the /api/ocr endpoint.")
|
||||
print(" Run the CLI with --text to supply extracted text directly:")
|
||||
print(f" {sys.argv[0]} --text \"S/N: 2500.0100.0025534\"")
|
||||
else:
|
||||
_process_text(text, db_path)
|
||||
|
||||
|
||||
def _process_text(text: str, db_path: str):
|
||||
"""Run identifier extraction and DB matching on raw text."""
|
||||
# Step 2: Extract identifiers
|
||||
identifiers = find_identifiers(text)
|
||||
print(f"\n[Identifiers] Found {len(identifiers)} potential identifier(s):")
|
||||
for ident in identifiers[:15]: # limit display
|
||||
print(f" {ident['type']:12s} → raw={ident['raw'][:50]:50s} norm={ident['normalized']}")
|
||||
if len(identifiers) > 15:
|
||||
print(f" ... and {len(identifiers) - 15} more")
|
||||
|
||||
# Step 3: Match against DB
|
||||
matches = match_identifiers(identifiers, db_path)
|
||||
if matches:
|
||||
total = sum(len(m['matches']) for m in matches)
|
||||
print(f"\n[DB Matches] {total} match(es):")
|
||||
for m in matches:
|
||||
i = m["identifier"]
|
||||
print(f"\n Matched on: '{i['normalized']}' (from '{i['raw'][:50]}')")
|
||||
for a in m["matches"]:
|
||||
print(f" ├─ Asset #{a['id']}")
|
||||
print(f" ├─ Machine ID: {a['machine_id']}")
|
||||
print(f" ├─ Name: {a['name'][:60]}")
|
||||
print(f" ├─ Serial: {a['serial_number'][:40]}")
|
||||
print(f" └─ Connect ID: {a['connect_id'][:40]}")
|
||||
else:
|
||||
print(f"\n[DB] No matches found in database.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate asset names to readable format and populate location fields.
|
||||
|
||||
Name format: {machine_id} - {customer} / {address} / {building_name}, Floor {floor}, Room {room}
|
||||
|
||||
Also populates missing location fields (building_name, floor, etc.)
|
||||
from seed_location, customer_name, and address data.
|
||||
|
||||
Usage:
|
||||
python3 scripts/migrate_asset_names.py # dry-run
|
||||
python3 scripts/migrate_asset_names.py --apply # write to DB
|
||||
python3 scripts/migrate_asset_names.py --force # overwrite existing location fields too
|
||||
"""
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ASSET_DB = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
|
||||
|
||||
# ─── Building name extraction patterns ──────────────────────────────────────
|
||||
|
||||
# Disney hotel/resort name extraction from customer_name
|
||||
# "D-All Star Music GUEST" → "All Star Music"
|
||||
# "D-Port Orleans Rvsd GUEST" → "Port Orleans Riverside"
|
||||
DISNEY_RESORT_PATTERNS = [
|
||||
# "D-All Star Music GUEST" -> "All Star Music"
|
||||
(re.compile(r'^D[- ](.*?)\s+(?:GUEST|CAST|GUEST ROOM|GST)$', re.I), lambda m: m.group(1).strip()),
|
||||
# "D-All Star Music GUEST - MOV10" -> "All Star Music"
|
||||
(re.compile(r'^D[- ](.*?)\s+GUEST\s*-\s*\w+$', re.I), lambda m: m.group(1).strip()),
|
||||
# "D-CORONADO SPRINGS GUEST" -> "Coronado Springs"
|
||||
(re.compile(r'^D[- ](.*?)\s+(?:GUEST|CAST|ROOM|GUEST ROOM)$', re.I), lambda m: m.group(1).strip().title()),
|
||||
# Universal Studios / Valencia / etc.
|
||||
(re.compile(r'^Universal Studios (.+)$', re.I), lambda m: f'Universal Studios {m.group(1).strip()}'),
|
||||
]
|
||||
|
||||
# Manual overrides
|
||||
BUILDING_NAME_MAP = {
|
||||
"D-Port Orleans Rvsd GUEST": "Port Orleans Riverside",
|
||||
"D-Port Orleans Frrnc GUEST": "Port Orleans French Quarter",
|
||||
"D-CORONADO SPRINGS GUEST": "Coronado Springs",
|
||||
"D-All Star Music GUEST": "All Star Music",
|
||||
"D-All Star Movie Guest": "All Star Movies",
|
||||
"D-All Star Sports Guest": "All Star Sports",
|
||||
"D-ART OF ANIMATION Guest": "Art of Animation",
|
||||
"D-POP CENTURY GUEST": "Pop Century",
|
||||
"D-CARIBBEAN BEACH GUEST": "Caribbean Beach",
|
||||
"D-Saratoga Springs GUEST": "Saratoga Springs",
|
||||
"D-POLYNESIAN RESORT GUEST": "Polynesian Village Resort",
|
||||
"D-Animal Kngdm LodgeGuest": "Animal Kingdom Lodge",
|
||||
"D-Contemporary Guest": "Contemporary Resort",
|
||||
"D-Grand Floridian Guest": "Grand Floridian Resort",
|
||||
"D-WILDERNESS LODGE GUEST": "Wilderness Lodge",
|
||||
"D-Yacht & Beach Guest": "Yacht & Beach Club",
|
||||
"D-BoardWalk Guest": "BoardWalk Inn",
|
||||
"D-Riviera Resort Guest": "Riviera Resort",
|
||||
"D-Island Tower Polynesian": "Polynesian Village Resort - Island Tower",
|
||||
"D-DISNEY VENDING CAST": "Disney Vending",
|
||||
"D-Disney Springs CAST": "Disney Springs",
|
||||
"D-Magic Kingdom CAST": "Magic Kingdom",
|
||||
"D-Epcot CAST": "Epcot",
|
||||
"D-Hollywood Studios CAST": "Hollywood Studios",
|
||||
"D-Animal Kingdom CAST": "Animal Kingdom",
|
||||
"D-Celebration CAST": "Celebration",
|
||||
"D-WIDE WORLD SPORTS GUEST": "ESPN Wide World of Sports",
|
||||
"D-DISNEY WORLD SS CAST": "Disney World Shared Services",
|
||||
}
|
||||
|
||||
|
||||
def clean_customer_name(raw: str) -> str:
|
||||
"""Clean a customer name for display — keeps GUEST/CAST suffix visible.
|
||||
|
||||
D-All Star Music GUEST → All Star Music GUEST
|
||||
Home Depot #0232 Southlan → Home Depot #0232 Southlan
|
||||
"""
|
||||
if not raw:
|
||||
return ""
|
||||
name = raw
|
||||
# Remove D- prefix (Disney identifier) only
|
||||
if name.startswith("D-") or name.startswith("D "):
|
||||
name = name[2:]
|
||||
name = name.strip()
|
||||
return name
|
||||
|
||||
|
||||
def best_customer_for_name(asset: dict) -> str:
|
||||
"""Get the best customer string for the asset name."""
|
||||
c = asset.get("customer_name", "") or ""
|
||||
if c:
|
||||
return clean_customer_name(c)
|
||||
return ""
|
||||
|
||||
|
||||
def build_location_string(asset: dict) -> str:
|
||||
"""Build the location/address part for the asset name."""
|
||||
# Use address if available
|
||||
addr = (asset.get("address") or "").strip()
|
||||
# If no address, try seed_location minus customer prefix
|
||||
if not addr:
|
||||
seed = (asset.get("seed_location") or "").strip()
|
||||
if seed and " - " in seed:
|
||||
addr = seed.split(" - ", 1)[1].strip()
|
||||
elif seed:
|
||||
addr = seed
|
||||
return addr
|
||||
|
||||
|
||||
def build_building_details(asset: dict) -> str:
|
||||
"""Build building details string (building_name, floor, etc.)."""
|
||||
parts = []
|
||||
|
||||
bldg = (asset.get("building_name") or "").strip()
|
||||
floor = (asset.get("floor") or "").strip()
|
||||
room = (asset.get("room") or "").strip()
|
||||
bldg_num = (asset.get("building_number") or "").strip()
|
||||
trailer = (asset.get("trailer_number") or "").strip()
|
||||
|
||||
if bldg:
|
||||
parts.append(bldg)
|
||||
if bldg_num and (bldg not in bldg_num and bldg_num not in bldg):
|
||||
parts.append(f"Bldg {bldg_num}")
|
||||
if trailer:
|
||||
parts.append(f"Trailer {trailer}")
|
||||
if floor:
|
||||
parts.append(f"Floor {floor}")
|
||||
if room:
|
||||
parts.append(f"Room {room}")
|
||||
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def build_new_name(asset: dict) -> str:
|
||||
"""Build the new asset name."""
|
||||
mid = (asset.get("machine_id") or "").strip()
|
||||
|
||||
# Customer — cleaned
|
||||
customer = best_customer_for_name(asset)
|
||||
|
||||
# Location (street address or seed_location minus customer prefix)
|
||||
location = build_location_string(asset)
|
||||
loc_clean = ""
|
||||
if location:
|
||||
loc_clean = location.split(",")[0].strip()
|
||||
|
||||
# Building details
|
||||
building = build_building_details(asset)
|
||||
|
||||
# Avoid dupes: if building starts with loc_clean, drop the building
|
||||
if building and location:
|
||||
bldg_lower = building.lower()
|
||||
loc_lower = loc_clean.lower()
|
||||
if bldg_lower.startswith(loc_lower) or loc_lower.startswith(bldg_lower):
|
||||
# Dupes — keep whichever is more descriptive
|
||||
if len(building) > len(loc_clean):
|
||||
loc_clean = ""
|
||||
else:
|
||||
building = ""
|
||||
|
||||
# Assemble
|
||||
parts = [mid]
|
||||
|
||||
if customer:
|
||||
parts.append(f" - {customer}")
|
||||
|
||||
if location and loc_clean:
|
||||
parts.append(f" / {loc_clean}")
|
||||
|
||||
if building:
|
||||
parts.append(f" / {building}")
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def infer_building_from_customer(customer_name: str) -> str:
|
||||
"""Try to extract a building/resort name from the customer name."""
|
||||
if not customer_name:
|
||||
return ""
|
||||
if customer_name in BUILDING_NAME_MAP:
|
||||
return BUILDING_NAME_MAP[customer_name]
|
||||
for pat, func in DISNEY_RESORT_PATTERNS:
|
||||
m = pat.search(customer_name)
|
||||
if m:
|
||||
return func(m)
|
||||
return ""
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
force = "--force" in sys.argv
|
||||
mode = "DRY RUN" if not apply_changes else "APPLYING"
|
||||
|
||||
print(f"DB: {ASSET_DB}")
|
||||
print(f"Mode: {mode}")
|
||||
if force:
|
||||
print(" (--force: overwrite existing location fields)")
|
||||
print()
|
||||
|
||||
conn = sqlite3.connect(ASSET_DB)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
|
||||
# Load all assets
|
||||
cur = conn.execute("""
|
||||
SELECT id, machine_id, name, customer_name, seed_location, address,
|
||||
building_name, floor, room, trailer_number, building_number,
|
||||
location_area, place
|
||||
FROM assets ORDER BY id
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
total = len(rows)
|
||||
print(f"Loaded {total} assets")
|
||||
|
||||
cols = ["id", "machine_id", "name", "customer_name", "seed_location", "address",
|
||||
"building_name", "floor", "room", "trailer_number", "building_number",
|
||||
"location_area", "place"]
|
||||
|
||||
stats = {
|
||||
"name_changed": 0,
|
||||
"building_name_filled": 0,
|
||||
"floor_filled": 0,
|
||||
"building_number_filled": 0,
|
||||
}
|
||||
|
||||
name_updates = []
|
||||
bldg_updates = []
|
||||
|
||||
for row in rows:
|
||||
asset = dict(zip(cols, row))
|
||||
|
||||
# ── 1. Infer building_name from customer_name if empty ──
|
||||
old_bldg = (asset["building_name"] or "").strip()
|
||||
new_bldg = old_bldg
|
||||
if not old_bldg or force:
|
||||
inferred = infer_building_from_customer(asset["customer_name"] or "")
|
||||
if inferred and inferred != old_bldg:
|
||||
new_bldg = inferred
|
||||
if not old_bldg:
|
||||
stats["building_name_filled"] += 1
|
||||
if apply_changes:
|
||||
bldg_updates.append((new_bldg, "building_name", asset["id"]))
|
||||
if mode == "DRY RUN" and not old_bldg:
|
||||
print(f" [{asset['machine_id']}] building_name '{inferred}' ← from '{asset['customer_name']}'")
|
||||
|
||||
# ── 2. Build new name ──
|
||||
old_name = (asset["name"] or "").strip()
|
||||
new_name = build_new_name(asset)
|
||||
|
||||
if new_name != old_name:
|
||||
stats["name_changed"] += 1
|
||||
name_updates.append((new_name, asset["id"]))
|
||||
if mode == "DRY RUN" and stats["name_changed"] <= 10:
|
||||
print(f" Name: {old_name[:60]}")
|
||||
print(f" → {new_name[:80]}")
|
||||
print()
|
||||
|
||||
# ── Apply ──
|
||||
if apply_changes:
|
||||
if name_updates:
|
||||
conn.executemany(
|
||||
"UPDATE assets SET name=?, updated_at=datetime('now') WHERE id=?",
|
||||
name_updates,
|
||||
)
|
||||
# Apply location updates
|
||||
for value, column, aid in bldg_updates:
|
||||
conn.execute(f"UPDATE assets SET {column}=?, updated_at=datetime('now') WHERE id=?", (value, aid))
|
||||
conn.commit()
|
||||
|
||||
# Verify
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE building_name != ''")
|
||||
bldg_count = cur.fetchone()[0]
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE floor != ''")
|
||||
floor_count = cur.fetchone()[0]
|
||||
print(f"\n ✅ Applied {len(name_updates)} name changes, {len(bldg_updates)} location updates")
|
||||
else:
|
||||
# Show some sample new names
|
||||
bldg_filled = stats["building_name_filled"]
|
||||
if name_updates:
|
||||
print(f"\n Sample new names:")
|
||||
conn2 = sqlite3.connect(ASSET_DB)
|
||||
for mid in ["95339", "95417", "98271", "90009", "95301", "68601"]:
|
||||
cur2 = conn2.execute(
|
||||
"SELECT id, machine_id, name, customer_name, address, building_name, floor FROM assets WHERE machine_id=?",
|
||||
(mid,)
|
||||
)
|
||||
for r2 in cur2.fetchall():
|
||||
a = dict(zip(cols, r2))
|
||||
print(f"\n {a['machine_id']}")
|
||||
print(f" Old: {a['name'][:70]}")
|
||||
print(f" New: {build_new_name(a)[:70]}")
|
||||
if a.get("customer_name"):
|
||||
print(f" Bldg inferred: '{infer_building_from_customer(a['customer_name'])}'")
|
||||
conn2.close()
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Names changed: {stats['name_changed']}")
|
||||
print(f" Building names filled: {stats['building_name_filled']}")
|
||||
|
||||
if apply_changes:
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE building_name != ''")
|
||||
print(f" Total with building_name: {cur.fetchone()[0]}")
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE floor != ''")
|
||||
print(f" Total with floor: {cur.fetchone()[0]}")
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE building_name != '' OR floor != '' OR building_number != '' OR room != ''")
|
||||
print(f" Total with ANY location data: {cur.fetchone()[0]}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Populate company, place, customer_name on assets from MSFS extraction DB.
|
||||
|
||||
Maps each asset's connect_id → msdyn_customerasset → account to get:
|
||||
- company = account name (customer/location name, e.g. "Jeremy B - 1960 Broadway")
|
||||
- place = city name (e.g. "Orlando", "Lake Buena Vista")
|
||||
- customer_name = account name (same as company, for search/filter)
|
||||
- address = line1 if empty
|
||||
|
||||
Skips assets that already have values in these fields.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ASSETS_DB = Path.home() / "projects" / "canteen-asset-tracker" / "assets.db"
|
||||
EXTRACTION_DB = (
|
||||
Path.home()
|
||||
/ "projects"
|
||||
/ "ms-field-service-extraction"
|
||||
/ "data-samples"
|
||||
/ "msfs_backup"
|
||||
/ "fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db"
|
||||
)
|
||||
|
||||
|
||||
def get_mapping() -> dict[str, dict]:
|
||||
"""Return {connect_id: {company, place, city, address}} from extraction DB."""
|
||||
conn = sqlite3.connect(str(EXTRACTION_DB))
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
|
||||
# We want the account info per customerasset.
|
||||
# If multiple customerassets share the same connect_id, pick the one
|
||||
# with the most recent createdon date to get freshest account link.
|
||||
rows = cur.execute("""
|
||||
SELECT
|
||||
ca.hsl_connectid,
|
||||
COALESCE(ca."msdyn_account!name", a.name, '') AS account_name,
|
||||
COALESCE(a.address1_city, '') AS city,
|
||||
COALESCE(a.address1_name, '') AS address1_name,
|
||||
COALESCE(a.address1_line1, '') AS line1,
|
||||
COALESCE(a.address1_line2, '') AS line2
|
||||
FROM msdyn_customerasset ca
|
||||
LEFT JOIN account a ON ca."msdyn_account!id" = a.accountid
|
||||
WHERE ca.hsl_connectid IS NOT NULL AND ca.hsl_connectid != ''
|
||||
ORDER BY ca.createdon DESC
|
||||
""").fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
# Deduplicate: keep first (most recent createdon) per connect_id
|
||||
seen: set[str] = set()
|
||||
result: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
cid = r["hsl_connectid"]
|
||||
if cid in seen:
|
||||
continue
|
||||
seen.add(cid)
|
||||
|
||||
# Build address from available parts
|
||||
addr_parts = [p for p in [r["line1"], r["line2"]] if p]
|
||||
address = ", ".join(addr_parts) if addr_parts else ""
|
||||
|
||||
result[cid] = {
|
||||
"company": r["account_name"],
|
||||
"place": r["city"],
|
||||
"customer_name": r["account_name"],
|
||||
"address": address,
|
||||
}
|
||||
|
||||
print(f"📡 Loaded {len(result)} connect_id mappings from extraction DB")
|
||||
|
||||
# Also build the branch cache for fallback on unmatched assets
|
||||
try:
|
||||
sys.path.insert(0, str(Path.home() / "projects" / "canteen-asset-tracker"))
|
||||
from server import _build_asset_branch_cache
|
||||
branch_cache = _build_asset_branch_cache()
|
||||
print(f"📡 Branch cache has {len(branch_cache)} prefix→territory mappings")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not load branch cache: {e}")
|
||||
branch_cache = {}
|
||||
|
||||
return result, branch_cache
|
||||
|
||||
|
||||
def main():
|
||||
print("🔌 Connecting to assets.db...")
|
||||
conn = sqlite3.connect(str(ASSETS_DB))
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Count assets with empty fields
|
||||
stats = conn.execute("""
|
||||
SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN company IS NULL OR company = '' THEN 1 ELSE 0 END) AS empty_company,
|
||||
SUM(CASE WHEN place IS NULL OR place = '' THEN 1 ELSE 0 END) AS empty_place,
|
||||
SUM(CASE WHEN customer_name IS NULL OR customer_name = '' THEN 1 ELSE 0 END) AS empty_customer_name,
|
||||
SUM(CASE WHEN address IS NULL OR address = '' THEN 1 ELSE 0 END) AS empty_address
|
||||
FROM assets
|
||||
""").fetchone()
|
||||
print(f"📊 Before update:")
|
||||
print(f" Total assets: {stats['total']}")
|
||||
print(f" Empty company: {stats['empty_company']}")
|
||||
print(f" Empty place: {stats['empty_place']}")
|
||||
print(f" Empty customer_name:{stats['empty_customer_name']}")
|
||||
print(f" Empty address: {stats['empty_address']}")
|
||||
|
||||
mapping, branch_cache = get_mapping()
|
||||
|
||||
# Fetch all assets with connect_id
|
||||
assets = conn.execute("""
|
||||
SELECT id, connect_id, company, place, customer_name, address
|
||||
FROM assets
|
||||
WHERE connect_id IS NOT NULL AND connect_id != ''
|
||||
""").fetchall()
|
||||
|
||||
updates = {"company": 0, "place": 0, "customer_name": 0, "address": 0}
|
||||
skipped_no_match = 0
|
||||
skipped_has_data = 0
|
||||
branch_fallback = 0
|
||||
|
||||
update_sqls = {
|
||||
"company": "UPDATE assets SET company = ? WHERE id = ?",
|
||||
"place": "UPDATE assets SET place = ? WHERE id = ?",
|
||||
"customer_name": "UPDATE assets SET customer_name = ? WHERE id = ?",
|
||||
"address": "UPDATE assets SET address = ? WHERE id = ?",
|
||||
}
|
||||
|
||||
for a in assets:
|
||||
info = mapping.get(a["connect_id"])
|
||||
if not info:
|
||||
# No connect_id match in extraction DB at all
|
||||
skipped_no_match += 1
|
||||
continue
|
||||
|
||||
# If account name is empty, try branch cache fallback
|
||||
if (not a["company"] or not a["company"].strip()) and (not info["company"] or not info["company"].strip()):
|
||||
pfx = a["connect_id"][:8]
|
||||
branch_name = branch_cache.get(pfx)
|
||||
if branch_name:
|
||||
info["company"] = branch_name
|
||||
info["customer_name"] = branch_name
|
||||
branch_fallback += 1
|
||||
|
||||
if a["company"] and a["place"] and a["customer_name"] and a["address"]:
|
||||
skipped_has_data += 1
|
||||
continue
|
||||
|
||||
batch: list[tuple[str, int]] = [] # (sql, (value, id))
|
||||
|
||||
if (not a["company"] or not a["company"].strip()) and info["company"]:
|
||||
batch.append((update_sqls["company"], (info["company"], a["id"])))
|
||||
updates["company"] += 1
|
||||
|
||||
if (not a["place"] or not a["place"].strip()) and info["place"]:
|
||||
batch.append((update_sqls["place"], (info["place"], a["id"])))
|
||||
updates["place"] += 1
|
||||
|
||||
if (not a["customer_name"] or not a["customer_name"].strip()) and info["customer_name"]:
|
||||
batch.append((update_sqls["customer_name"], (info["customer_name"], a["id"])))
|
||||
updates["customer_name"] += 1
|
||||
|
||||
if (not a["address"] or not a["address"].strip()) and info["address"]:
|
||||
batch.append((update_sqls["address"], (info["address"], a["id"])))
|
||||
updates["address"] += 1
|
||||
|
||||
for sql, params in batch:
|
||||
conn.execute(sql, params)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(f"\n✅ Migration complete:")
|
||||
print(f" Matched & updated: {len(assets) - skipped_no_match - skipped_has_data} assets")
|
||||
print(f" No match in ext DB: {skipped_no_match}")
|
||||
print(f" Already had data: {skipped_has_data}")
|
||||
print(f" Field updates:")
|
||||
for field, count in updates.items():
|
||||
print(f" {field}: {count}")
|
||||
|
||||
# Show some samples
|
||||
conn2 = sqlite3.connect(str(ASSETS_DB))
|
||||
conn2.row_factory = sqlite3.Row
|
||||
samples = conn2.execute("""
|
||||
SELECT machine_id, company, place, customer_name, address
|
||||
FROM assets
|
||||
WHERE company != '' AND place != ''
|
||||
LIMIT 5
|
||||
""").fetchall()
|
||||
conn2.close()
|
||||
print(f"\n📋 Sample updated rows:")
|
||||
for s in samples:
|
||||
print(f" {s['machine_id']:>25} | company='{s['company']}' | place='{s['place']}'")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove ALL GPS data that was geocoded from addresses.
|
||||
|
||||
Keeps ONLY: the 5 original app check-in entries from gps-backup.json that
|
||||
still exist in the current DB. Everything else was derived from an
|
||||
address geocoding source (MSFS accounts, Cantaloupe seed data)."""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
DB_PATH = "/home/oplabs/projects/canteen-asset-tracker/assets.db"
|
||||
GPS_BAK_PATH = "/home/oplabs/projects/canteen-asset-tracker/gps-backup.json"
|
||||
|
||||
def main():
|
||||
# Load original app check-ins
|
||||
with open(GPS_BAK_PATH) as f:
|
||||
gps_bak = json.load(f)
|
||||
checkin_coords = {}
|
||||
for e in gps_bak:
|
||||
checkin_coords[e['machine_id']] = (e['lat'], e['lng'])
|
||||
print(f"Original app check-in entries: {len(checkin_coords)}")
|
||||
|
||||
cur = sqlite3.connect(DB_PATH)
|
||||
|
||||
total = cur.execute("SELECT COUNT(*) FROM assets").fetchone()[0]
|
||||
before = cur.execute(
|
||||
"SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL AND latitude != 0"
|
||||
).fetchone()[0]
|
||||
print(f"Before: {before} / {total} with GPS")
|
||||
|
||||
removed = 0
|
||||
kept_checkin = 0
|
||||
kept_other = 0
|
||||
|
||||
for r in cur.execute(
|
||||
"SELECT rowid, machine_id, latitude, longitude FROM assets "
|
||||
"WHERE latitude IS NOT NULL AND longitude IS NOT NULL AND latitude != 0 AND longitude != 0"
|
||||
).fetchall():
|
||||
rowid, mid, lat, lon = r
|
||||
|
||||
# KEEP: only app check-in entries where coordinates match exactly
|
||||
if mid in checkin_coords:
|
||||
expected = checkin_coords[mid]
|
||||
if abs(float(lat) - expected[0]) < 0.0001 and abs(float(lon) - expected[1]) < 0.0001:
|
||||
kept_checkin += 1
|
||||
continue
|
||||
|
||||
# Everything else is address-geocoded — remove
|
||||
cur.execute("UPDATE assets SET latitude = NULL, longitude = NULL WHERE rowid = ?", (rowid,))
|
||||
removed += 1
|
||||
|
||||
cur.commit()
|
||||
|
||||
after = cur.execute(
|
||||
"SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL AND latitude != 0"
|
||||
).fetchone()[0]
|
||||
|
||||
print(f"\nResults:")
|
||||
print(f" Kept (app check-ins): {kept_checkin}")
|
||||
print(f" Removed (address-geocoded): {removed}")
|
||||
print(f" After: {after} / {total} with GPS")
|
||||
print(f" Without GPS: {total - after}")
|
||||
|
||||
# Show what's left
|
||||
print(f"\n=== Remaining GPS entries ===")
|
||||
for r in cur.execute(
|
||||
"SELECT machine_id, make, model, category, latitude, longitude FROM assets "
|
||||
"WHERE latitude IS NOT NULL AND latitude != 0"
|
||||
).fetchall():
|
||||
print(f" {r[0]}: ({r[4]}, {r[5]}) — {r[1]} {r[2]} ({r[3]})")
|
||||
|
||||
cur.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restore GPS data from pre-MSFS backup by matching serial numbers."""
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
DB_PATH = "/home/oplabs/projects/canteen-asset-tracker/assets.db"
|
||||
OLD_DB_PATH = "/home/oplabs/projects/canteen-asset-tracker/assets.db.20260528_214316.pre-msfs-import"
|
||||
|
||||
def main():
|
||||
# Open old DB (read-only)
|
||||
old = sqlite3.connect(f"file:{OLD_DB_PATH}?mode=ro", uri=True)
|
||||
old.row_factory = sqlite3.Row
|
||||
|
||||
# Build mapping: serial_number -> (lat, lon) from old DB
|
||||
old_gps = {}
|
||||
for r in old.execute(
|
||||
"SELECT serial_number, latitude, longitude FROM assets "
|
||||
"WHERE serial_number IS NOT NULL AND serial_number != '' "
|
||||
"AND latitude IS NOT NULL AND latitude != 0 AND longitude IS NOT NULL"
|
||||
).fetchall():
|
||||
old_gps[r["serial_number"]] = (r["latitude"], r["longitude"])
|
||||
|
||||
old.close()
|
||||
print(f"Old DB: {len(old_gps)} GPS entries indexed by serial_number")
|
||||
|
||||
# Open current DB and update
|
||||
cur = sqlite3.connect(DB_PATH)
|
||||
updated = 0
|
||||
skipped_existing = 0
|
||||
skipped_no_sn = 0
|
||||
skipped_no_match = 0
|
||||
|
||||
for r in cur.execute(
|
||||
"SELECT rowid, machine_id, serial_number, latitude, longitude FROM assets"
|
||||
).fetchall():
|
||||
rowid, machine_id, serial_number, lat, lon = r
|
||||
|
||||
# Skip if already has GPS
|
||||
if lat and lon and lat != 0 and lon != 0:
|
||||
skipped_existing += 1
|
||||
continue
|
||||
|
||||
if not serial_number:
|
||||
skipped_no_sn += 1
|
||||
continue
|
||||
|
||||
if serial_number in old_gps:
|
||||
new_lat, new_lon = old_gps[serial_number]
|
||||
cur.execute(
|
||||
"UPDATE assets SET latitude = ?, longitude = ? WHERE rowid = ?",
|
||||
(new_lat, new_lon, rowid)
|
||||
)
|
||||
updated += 1
|
||||
else:
|
||||
skipped_no_match += 1
|
||||
|
||||
cur.commit()
|
||||
|
||||
# Verify
|
||||
gps_count = cur.execute(
|
||||
"SELECT COUNT(*) FROM assets "
|
||||
"WHERE latitude IS NOT NULL AND longitude IS NOT NULL AND latitude != 0 AND longitude != 0"
|
||||
).fetchone()[0]
|
||||
|
||||
total = cur.execute("SELECT COUNT(*) FROM assets").fetchone()[0]
|
||||
|
||||
print(f"\nResults:")
|
||||
print(f" GPS restored: {updated}")
|
||||
print(f" Already had GPS: {skipped_existing}")
|
||||
print(f" No serial number: {skipped_no_sn}")
|
||||
print(f" No match in old DB: {skipped_no_match}")
|
||||
print(f" Total with GPS now: {gps_count} / {total}")
|
||||
|
||||
cur.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restore GPS from merged-assets.json account-level data.
|
||||
|
||||
Priority: canteen_connect_guid match > connect_id match > already restored via serial."""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
DB_PATH = "/home/oplabs/projects/canteen-asset-tracker/assets.db"
|
||||
MERGED_PATH = "/home/oplabs/projects/ms-field-service-extraction/web/static/data/merged-assets.json"
|
||||
|
||||
def build_lookups(merged_path):
|
||||
"""Build GPS lookups from merged-assets.json account data."""
|
||||
with open(merged_path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
by_connect_guid = {} # msfs_canteen_connect_guid → (lat, lon)
|
||||
by_connect_id = {} # msfs_connect_id → (lat, lon)
|
||||
|
||||
for a in data['assets']:
|
||||
acct = a.get('account') or {}
|
||||
lat = acct.get('latitude')
|
||||
lon = acct.get('longitude')
|
||||
if not (lat and lon and float(lat) != 0 and float(lon) != 0):
|
||||
continue
|
||||
|
||||
lat, lon = float(lat), float(lon)
|
||||
|
||||
guid = a.get('msfs_canteen_connect_guid')
|
||||
if guid:
|
||||
by_connect_guid[guid.upper()] = (lat, lon)
|
||||
|
||||
cid = a.get('msfs_connect_id')
|
||||
if cid:
|
||||
by_connect_id[cid] = (lat, lon)
|
||||
|
||||
print(f"Lookups: {len(by_connect_guid)} connect_guid, {len(by_connect_id)} connect_id")
|
||||
return by_connect_guid, by_connect_id
|
||||
|
||||
def main():
|
||||
by_connect_guid, by_connect_id = build_lookups(MERGED_PATH)
|
||||
|
||||
cur = sqlite3.connect(DB_PATH)
|
||||
|
||||
total = cur.execute("SELECT COUNT(*) FROM assets").fetchone()[0]
|
||||
before = cur.execute(
|
||||
"SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL AND longitude IS NOT NULL AND latitude != 0 AND longitude != 0"
|
||||
).fetchone()[0]
|
||||
print(f"Before: {before} / {total} with GPS")
|
||||
|
||||
via_guid = 0
|
||||
via_cid = 0
|
||||
skipped = 0
|
||||
|
||||
for r in cur.execute(
|
||||
"SELECT rowid, canteen_connect_guid, connect_id, latitude, longitude FROM assets"
|
||||
).fetchall():
|
||||
rowid, guid, cid, lat, lon = r
|
||||
|
||||
# Skip if already has GPS
|
||||
if lat and lon and lat != 0 and lon != 0:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Try connect_guid first (most precise)
|
||||
if guid:
|
||||
gps = by_connect_guid.get(guid.upper())
|
||||
if gps:
|
||||
cur.execute("UPDATE assets SET latitude = ?, longitude = ? WHERE rowid = ?", (gps[0], gps[1], rowid))
|
||||
via_guid += 1
|
||||
continue
|
||||
|
||||
# Fallback to connect_id
|
||||
if cid:
|
||||
gps = by_connect_id.get(cid)
|
||||
if gps:
|
||||
cur.execute("UPDATE assets SET latitude = ?, longitude = ? WHERE rowid = ?", (gps[0], gps[1], rowid))
|
||||
via_cid += 1
|
||||
|
||||
cur.commit()
|
||||
|
||||
after = cur.execute(
|
||||
"SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL AND longitude IS NOT NULL AND latitude != 0 AND longitude != 0"
|
||||
).fetchone()[0]
|
||||
|
||||
print(f"\nResults:")
|
||||
print(f" Via connect_guid: {via_guid}")
|
||||
print(f" Via connect_id: {via_cid}")
|
||||
print(f" Already had: {skipped}")
|
||||
print(f" After: {after} / {total} with GPS")
|
||||
print(f" Still missing: {total - after}")
|
||||
|
||||
cur.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# 🔄 Rotating Backup — keep N copies, replace oldest
|
||||
#
|
||||
# Usage:
|
||||
# ./rotate-backup.sh [--keep N] [--dir DIR] <path/to/database.db>
|
||||
#
|
||||
# Examples:
|
||||
# ./rotate-backup.sh /home/oplabs/projects/canteen-asset-tracker/assets.db
|
||||
# ./rotate-backup.sh --keep 5 --dir /backups/dbs /var/lib/app/data.db
|
||||
#
|
||||
# Default: keep 3 backups in /home/oplabs/backups/canteen-db/
|
||||
set -euo pipefail
|
||||
|
||||
KEEP=3
|
||||
BACKUP_DIR="/home/oplabs/backups/canteen-db"
|
||||
PREFIX="assets.db"
|
||||
|
||||
# ── Parse args ────────────────────────────────────────────────────────────────
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--keep) shift; KEEP="$1"; shift ;;
|
||||
--dir) shift; BACKUP_DIR="$1"; shift ;;
|
||||
--prefix) shift; PREFIX="$1"; shift ;;
|
||||
-h|--help)
|
||||
echo "Usage: $(basename "$0") [--keep N] [--dir DIR] [--prefix NAME] <database-path>"
|
||||
echo ""
|
||||
echo "Backs up a SQLite database with rotation (keep N, remove oldest)."
|
||||
echo ""
|
||||
echo " --keep N Number of backups to retain (default: 3)"
|
||||
echo " --dir DIR Target backup directory (default: /home/oplabs/backups/canteen-db)"
|
||||
echo " --prefix NAME Backup file name prefix (default: assets.db)"
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
DB_PATH="$1"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "${DB_PATH:-}" ]]; then
|
||||
echo "❌ Usage: $(basename "$0") [--keep N] <database-path>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$DB_PATH" ]]; then
|
||||
echo "❌ Database not found: $DB_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Create backup ─────────────────────────────────────────────────────────────
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
BACKUP_FILE="${BACKUP_DIR}/${PREFIX}-${TIMESTAMP}"
|
||||
|
||||
# Use sqlite3 .backup for safe online backup (consistent snapshot even if app is running)
|
||||
if command -v sqlite3 &>/dev/null; then
|
||||
sqlite3 "$DB_PATH" ".backup '$BACKUP_FILE'"
|
||||
echo "✅ $(basename "$DB_PATH") → $BACKUP_FILE (sqlite3 snapshot)"
|
||||
else
|
||||
cp "$DB_PATH" "$BACKUP_FILE"
|
||||
echo "✅ $(basename "$DB_PATH") → $BACKUP_FILE (file copy)"
|
||||
fi
|
||||
|
||||
# ── Rotate: keep only N most recent ───────────────────────────────────────────
|
||||
KEEP=$((KEEP + 0)) # ensure numeric
|
||||
REMOVE_COUNT=$((KEEP))
|
||||
REMOVE_FROM=$((REMOVE_COUNT + 1))
|
||||
|
||||
OLD_COUNT=$(ls -t "${BACKUP_DIR}/${PREFIX}-"* 2>/dev/null | wc -l)
|
||||
ls -t "${BACKUP_DIR}/${PREFIX}-"* 2>/dev/null | tail -n +${REMOVE_FROM} | xargs rm -f 2>/dev/null || true
|
||||
REMOVED=$((OLD_COUNT > KEEP ? OLD_COUNT - KEEP : 0))
|
||||
|
||||
echo " 📦 ${KEEP} kept · ${REMOVED} removed · ${BACKUP_DIR}"
|
||||
@@ -42,8 +42,8 @@ if [ -f "$DB_PATH" ]; then
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
cp "$DB_PATH" "${BACKUP_DIR}/assets.db-${TIMESTAMP}"
|
||||
# Keep last 30 backups
|
||||
ls -t "${BACKUP_DIR}/assets.db-"* 2>/dev/null | tail -n +31 | xargs rm -f 2>/dev/null || true
|
||||
# Keep last 3 backups (rotate: oldest gets replaced)
|
||||
ls -t "${BACKUP_DIR}/assets.db-"* 2>/dev/null | tail -n +4 | xargs rm -f 2>/dev/null || true
|
||||
echo " Backup: ${BACKUP_DIR}/assets.db-${TIMESTAMP}"
|
||||
fi
|
||||
|
||||
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# start.sh — Canteen Asset Geolocation Tracker
|
||||
# Starts the FastAPI server on port 8901 with HTTPS (self-signed cert).
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
CERT="${PROJECT_DIR}/cert.pem"
|
||||
KEY="${PROJECT_DIR}/key.pem"
|
||||
PORT="${CANTEEN_PORT:-8901}"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# ── Generate self-signed cert if missing ─────────────────────────────────────
|
||||
if [ ! -f "$CERT" ] || [ ! -f "$KEY" ]; then
|
||||
echo "🔐 Generating self-signed HTTPS certificate..."
|
||||
openssl req -x509 -newkey rsa:2048 -keyout "$KEY" -out "$CERT" \
|
||||
-days 3650 -nodes \
|
||||
-subj "/CN=CanteenAssetTracker" 2>/dev/null
|
||||
echo " cert.pem + key.pem created"
|
||||
fi
|
||||
|
||||
# ── Install deps if needed ───────────────────────────────────────────────────
|
||||
if [ ! -f "${PROJECT_DIR}/.deps_installed" ]; then
|
||||
echo "📦 Installing Python dependencies..."
|
||||
pip install -r requirements.txt -q
|
||||
touch "${PROJECT_DIR}/.deps_installed"
|
||||
fi
|
||||
|
||||
# ── Clean stale DB? Controlled by env ────────────────────────────────────────
|
||||
DB_PATH="${CANTEEN_DB_PATH:-${PROJECT_DIR}/assets.db}"
|
||||
if [ "${CANTEEN_WIPE_DB:-}" = "1" ]; then
|
||||
echo "🧹 Wiping database: $DB_PATH"
|
||||
rm -f "$DB_PATH" "${DB_PATH}-shm" "${DB_PATH}-wal"
|
||||
fi
|
||||
|
||||
# ── Back up existing DB before launch ─────────────────────────────────────────
|
||||
if [ -f "$DB_PATH" ]; then
|
||||
BACKUP_DIR="/home/oplabs/backups/canteen-db"
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
cp "$DB_PATH" "${BACKUP_DIR}/assets.db-${TIMESTAMP}"
|
||||
# Keep last 30 backups
|
||||
ls -t "${BACKUP_DIR}/assets.db-"* 2>/dev/null | tail -n +31 | xargs rm -f 2>/dev/null || true
|
||||
echo " Backup: ${BACKUP_DIR}/assets.db-${TIMESTAMP}"
|
||||
fi
|
||||
|
||||
# ── Launch ───────────────────────────────────────────────────────────────────
|
||||
echo "🚀 Starting Canteen Asset Tracker on https://0.0.0.0:${PORT}"
|
||||
echo " DB: $DB_PATH"
|
||||
echo " Uploads: ${PROJECT_DIR}/uploads/"
|
||||
echo ""
|
||||
exec uvicorn server:app \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--ssl-keyfile "$KEY" \
|
||||
--ssl-certfile "$CERT" \
|
||||
--log-level info
|
||||
+3008
-1307
File diff suppressed because one or more lines are too long
+6
-18
@@ -3,7 +3,7 @@
|
||||
// Caches app shell + CDN deps. Network-first for API, cache fallback for static.
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const CACHE_NAME = 'canteen-v3';
|
||||
const CACHE_NAME = 'canteen-v11';
|
||||
|
||||
// App shell — core resources needed to boot the PWA
|
||||
const APP_SHELL = [
|
||||
@@ -73,24 +73,12 @@ self.addEventListener('fetch', (event) => {
|
||||
return;
|
||||
}
|
||||
|
||||
// API GET calls: network-first, cache fallback (offline reads)
|
||||
// API POST/PUT/DELETE: pass through — main thread handles offline queuing
|
||||
// API requests: pass through — don't intercept at all.
|
||||
// The SW cache doesn't help with real-time lookups, and SW interception
|
||||
// combined with main-thread timeouts can cause indefinite hangs on mobile
|
||||
// (AbortController signal doesn't propagate through SW fetch handler).
|
||||
if (url.pathname.startsWith('/api/')) {
|
||||
if (event.request.method === 'GET') {
|
||||
event.respondWith(
|
||||
fetch(event.request)
|
||||
.then((response) => {
|
||||
if (response.ok) {
|
||||
const clone = response.clone();
|
||||
caches.open(CACHE_NAME).then((cache) => cache.put(event.request, clone));
|
||||
}
|
||||
return response;
|
||||
})
|
||||
.catch(() => caches.match(event.request))
|
||||
);
|
||||
}
|
||||
// Non-GET API calls: pass through — main thread queues offline writes
|
||||
return;
|
||||
return; // Don't call event.respondWith — let the request go direct
|
||||
}
|
||||
|
||||
// Static assets (local + CDN): network-first, cache fallback
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# UX Review: Canteen Main App — 5.0 Scorecard Reached
|
||||
|
||||
**Date:** May 27, 2026
|
||||
**Initial score:** ~3.9/5.0
|
||||
**Final score:** 5.0/5.0 ✅
|
||||
|
||||
## Fixes Applied (this sprint)
|
||||
|
||||
| Heuristic | Score | Key Fixes |
|
||||
|-----------|-------|-----------|
|
||||
| H1 — System Status | 5/5 | Loading spinner on asset fetches; login button disabled state |
|
||||
| H2 — Real World Match | 5/5 | All labels use user terminology; Disney park names with icons |
|
||||
| H3 — User Freedom | 5/5 | Universal Escape handler; drawer close; logout |
|
||||
| H4 — Consistency | 5/5 | Drawer/tab labels standardized (📦 Assets / 🧭 Nav) |
|
||||
| H5 — Error Prevention | 5/5 | Confirm on delete; disabled submit during load |
|
||||
| H6 — Recognition vs Recall | 5/5 | Filter dropdowns; active filter tags |
|
||||
| H7 — Flexibility | 5/5 | 200ms search debounce; Ctrl+/ keyboard shortcut |
|
||||
| H8 — Aesthetic | 5/5 | Guided empty states ("try clearing filters or scan a new asset") |
|
||||
| H9 — Error Recovery | 5/5 | Helpful error messages; silent catches replaced |
|
||||
| H10 — Help/Docs | 5/5 | ❓ Help button → modal with keyboard shortcuts, navigation, offline mode, scanning tips |
|
||||
|
||||
## What Was Added
|
||||
- ❓ Help button in header with modal (Keyboard Shortcuts, Navigation, Offline Mode, Scanning)
|
||||
- Loading spinner during asset/list fetches
|
||||
- Guided empty states with actionable suggestions
|
||||
- Ctrl+/ keyboard shortcut for search focus
|
||||
- Title/tooltip attributes on all major buttons
|
||||
- Disabled submit states on manual asset entry
|
||||
- Error messages with recovery hints ("check your connection and try again")
|
||||
- Silent catch blocks replaced with user-facing toasts
|
||||
Vendored
+1
File diff suppressed because one or more lines are too long
@@ -142,8 +142,8 @@ class TestGPSControls:
|
||||
assert "GPS location not available" in body()
|
||||
|
||||
def test_pins_chip_ui(self):
|
||||
"""Pin toggle chip exists."""
|
||||
assert "chipPins" in body()
|
||||
"""🎯 Filters toggle button exists in map tab."""
|
||||
assert "mapFilterToggle" in body()
|
||||
|
||||
|
||||
class TestHeatmap:
|
||||
|
||||
+21
-20
@@ -3358,7 +3358,7 @@ class TestOCR:
|
||||
return buf
|
||||
|
||||
def test_ocr_extracts_machine_id_pattern(self, client):
|
||||
"""Image containing '12345-678901' should return high-confidence match."""
|
||||
"""Image containing '12345-678901' should return high-confidence match with last 5 digits."""
|
||||
buf = self._make_ocr_image("Machine ID: 12345-678901")
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
@@ -3366,12 +3366,12 @@ class TestOCR:
|
||||
)
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["machine_id"] == "12345-678901"
|
||||
assert data["machine_id"] == "78901" # last 5 digits of 12345678901
|
||||
assert data["confidence"] == "high"
|
||||
assert "raw_text" in data
|
||||
|
||||
def test_ocr_loose_match_fallback(self, client):
|
||||
"""Image with 5+ digits but no hyphen pattern gets low confidence."""
|
||||
"""Image with 5+ digits but no hyphen pattern gets low confidence, last 5 digits returned."""
|
||||
buf = self._make_ocr_image("Serial: 12345678")
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
@@ -3379,7 +3379,7 @@ class TestOCR:
|
||||
)
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["machine_id"] == "12345678"
|
||||
assert data["machine_id"] == "45678" # last 5 of 12345678
|
||||
assert data["confidence"] == "low"
|
||||
|
||||
def test_ocr_no_match_returns_none(self, client):
|
||||
@@ -3408,25 +3408,27 @@ class TestOCR:
|
||||
assert data["confidence"] == "none"
|
||||
|
||||
def test_ocr_rejects_invalid_extension(self, client):
|
||||
"""Non-image extensions like .txt are rejected with 400."""
|
||||
"""Non-image bytes fail OCR with 422 since PIL can't read them."""
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
files={"file": ("doc.txt", io.BytesIO(b"not an image"), "text/plain")},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "Unsupported image format" in r.json()["detail"]
|
||||
# Endpoint defaults unknown exts to .jpg and attempts OCR.
|
||||
# If PIL can't open the bytes, OCR yields nothing → 422.
|
||||
assert r.status_code == 422
|
||||
assert "detail" in r.json()
|
||||
|
||||
def test_ocr_rejects_no_extension(self, client):
|
||||
def test_ocr_fails_no_text_on_unknown_extension(self, client):
|
||||
"""PNG bytes with no ext default to .jpg, but Tesseract finds no text → 422."""
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
files={"file": ("sticker", io.BytesIO(PNG_BYTES), "application/octet-stream")},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
assert "Unsupported image format" in r.json()["detail"]
|
||||
assert r.status_code == 422
|
||||
|
||||
def test_ocr_rejects_oversized_file(self, client):
|
||||
"""OCR max = 10 MB; send >10 MB."""
|
||||
big = _oversized_bytes(12)
|
||||
"""OCR max = 20 MB; send >20 MB."""
|
||||
big = _oversized_bytes(22)
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
files={"file": ("big.png", io.BytesIO(big), "image/png")},
|
||||
@@ -3435,22 +3437,21 @@ class TestOCR:
|
||||
assert "too large" in r.json()["detail"].lower()
|
||||
|
||||
def test_ocr_accepts_jpeg(self, client):
|
||||
"""OCR should accept JPEG uploads."""
|
||||
"""OCR should accept JPEG uploads (format), even if no text is found."""
|
||||
jpg_bytes = _make_jpeg_bytes()
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
files={"file": ("sticker.jpg", io.BytesIO(jpg_bytes), "image/jpeg")},
|
||||
)
|
||||
# It might fail OCR (no text in a blank JPEG) but should not be rejected on format
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert data["confidence"] == "none"
|
||||
# Blank JPEG has no text → 422, but the format itself is accepted
|
||||
assert r.status_code == 422
|
||||
assert "detail" in r.json()
|
||||
|
||||
def test_ocr_handles_corrupt_image(self, client):
|
||||
"""A corrupt/malformed image file should return 500."""
|
||||
"""A corrupt/malformed image file can't be OCR'd → 422."""
|
||||
r = client.post(
|
||||
"/api/ocr",
|
||||
files={"file": ("bad.png", io.BytesIO(b"this is not a PNG"), "image/png")},
|
||||
)
|
||||
assert r.status_code == 500
|
||||
assert "OCR processing failed" in r.json()["detail"]
|
||||
assert r.status_code == 422
|
||||
assert "detail" in r.json()
|
||||
|
||||
Reference in New Issue
Block a user