feat: asset naming, location extraction, and location display
- New asset naming: {MID} - {Customer} / {Address} / {Building details}
- Extracted 660 building_names from customer data (Disney resorts, hotels)
- 1,704 assets now have location data (building_name, floor, etc.)
- Location section in detail view now shows building, floor, trailer, room
- Added has_gps filter to API for map loading
- Added location_source column to track GPS origin
This commit is contained in:
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Apply GPS coordinates from multiple sources to canteen assets DB.
|
||||
|
||||
Sources (priority order):
|
||||
1. Booking GPS (MSFS technician check-in) — most reliable
|
||||
2. Photo EXIF GPS (OCR'd technician photos)
|
||||
3. exif-test-dev GPS (iPhone uploads)
|
||||
|
||||
Usage:
|
||||
python3 scripts/apply_gps_sources.py # dry-run
|
||||
python3 scripts/apply_gps_sources.py --apply # write to DB
|
||||
python3 scripts/apply_gps_sources.py --force # overwrite existing GPS too
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ASSET_DB = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
MSFS_DATA_DIR = "/home/oplabs/projects/ms-field-service-extraction/web/static/data"
|
||||
MSFS_BACKUP_DB = "/home/oplabs/projects/ms-field-service-extraction/data-samples/msfs_backup/fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db"
|
||||
|
||||
|
||||
def load_merged_assets() -> dict:
|
||||
"""Load merged MSFS<->canteen asset mappings. Returns guid->machine_id."""
|
||||
with open(f"{MSFS_DATA_DIR}/merged-assets.json") as f:
|
||||
merged = json.load(f)
|
||||
guid_to_mid = {}
|
||||
for a in merged["assets"]:
|
||||
cmid = a.get("canteen_machine_id")
|
||||
guid = a.get("msfs_canteen_connect_guid", "")
|
||||
if cmid and guid:
|
||||
guid_to_mid[guid.upper()] = cmid
|
||||
return guid_to_mid
|
||||
|
||||
|
||||
def get_booking_gps(guid_to_mid: dict) -> dict:
|
||||
"""Extract booking GPS from MSFS backup, grouped by machine_id."""
|
||||
db = sqlite3.connect(MSFS_BACKUP_DB)
|
||||
|
||||
# Work order -> customer asset
|
||||
wo_to_asset = {}
|
||||
for w in db.execute(
|
||||
'SELECT "msdyn_workorderid", "msdyn_customerasset!id" FROM "msdyn_workorder" WHERE "msdyn_customerasset!id" IS NOT NULL'
|
||||
):
|
||||
wo_to_asset[w[0]] = w[1]
|
||||
|
||||
# Customer asset -> GUID
|
||||
asset_info = {}
|
||||
for c in db.execute(
|
||||
'SELECT "msdyn_customerassetid", "hsl_canteenconnectguid" FROM "msdyn_customerasset"'
|
||||
):
|
||||
asset_info[c[0]] = (c[1] or "").upper() if c[1] else ""
|
||||
|
||||
# Collect booking GPS points
|
||||
booking_by_mid = defaultdict(list)
|
||||
for r in db.execute(
|
||||
"""
|
||||
SELECT b."msdyn_latitude", b."msdyn_longitude", b."msdyn_workorder!id"
|
||||
FROM "bookableresourcebooking" b
|
||||
WHERE b."msdyn_latitude" IS NOT NULL AND b."msdyn_longitude" IS NOT NULL
|
||||
"""
|
||||
):
|
||||
lat, lon, wo_id = r
|
||||
if not wo_id or wo_id not in wo_to_asset:
|
||||
continue
|
||||
asset_id = wo_to_asset[wo_id]
|
||||
if not asset_id or asset_id not in asset_info:
|
||||
continue
|
||||
guid = asset_info[asset_id]
|
||||
if guid in guid_to_mid:
|
||||
mid = guid_to_mid[guid]
|
||||
booking_by_mid[mid].append((lat, lon))
|
||||
|
||||
# Average per machine
|
||||
result = {}
|
||||
for mid, points in booking_by_mid.items():
|
||||
result[mid] = {
|
||||
"lat": sum(p[0] for p in points) / len(points),
|
||||
"lon": sum(p[1] for p in points) / len(points),
|
||||
"source": "booking",
|
||||
"count": len(points),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def get_photo_gps() -> dict:
|
||||
"""Extract GPS from OCR'd technician photos."""
|
||||
try:
|
||||
with open(f"{MSFS_DATA_DIR}/ocr_results.json") as f:
|
||||
ocr = json.load(f)
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
|
||||
photo_by_mid = defaultdict(list)
|
||||
for k, v in ocr.items():
|
||||
if v.get("has_gps") and v.get("machine_id") and v.get("gps_lat") and v.get("gps_lon"):
|
||||
mid = v["machine_id"]
|
||||
# Normalize: strip leading zeros from last part
|
||||
parts = mid.split("-")
|
||||
last_part = parts[-1]
|
||||
if last_part.startswith("0") and len(last_part) > 5:
|
||||
last_part = last_part.lstrip("0")
|
||||
photo_by_mid[last_part].append((v["gps_lat"], v["gps_lon"]))
|
||||
|
||||
result = {}
|
||||
for mid, points in photo_by_mid.items():
|
||||
result[mid] = {
|
||||
"lat": sum(p[0] for p in points) / len(points),
|
||||
"lon": sum(p[1] for p in points) / len(points),
|
||||
"source": "photo",
|
||||
"count": len(points),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def get_exif_test_gps() -> dict:
|
||||
"""Extract GPS from exif-test-dev iPhone uploads."""
|
||||
exif_dev_db = str(
|
||||
Path.home() / "projects" / "exif-test-dev" / "photos.db"
|
||||
)
|
||||
if not Path(exif_dev_db).exists():
|
||||
return {}
|
||||
|
||||
db = sqlite3.connect(exif_dev_db)
|
||||
result = {}
|
||||
for r in db.execute(
|
||||
"SELECT machine_id, gps_lat, gps_lng FROM photos WHERE machine_id IS NOT NULL AND gps_lat IS NOT NULL"
|
||||
):
|
||||
mid, lat, lng = r
|
||||
if mid not in result:
|
||||
result[mid] = {"points": [], "source": "exif_dev"}
|
||||
result[mid]["points"].append((lat, lng))
|
||||
|
||||
averaged = {}
|
||||
for mid, data in result.items():
|
||||
pts = data["points"]
|
||||
averaged[mid] = {
|
||||
"lat": sum(p[0] for p in pts) / len(pts),
|
||||
"lon": sum(p[1] for p in pts) / len(pts),
|
||||
"source": "exif_dev",
|
||||
"count": len(pts),
|
||||
}
|
||||
return averaged
|
||||
|
||||
|
||||
def get_seed_gps(conn: sqlite3.Connection) -> dict:
|
||||
"""Extract GPS from Cantaloupe seed data if available."""
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"SELECT machine_id, json_extract(data, '$.Latitude'), json_extract(data, '$.Longitude') "
|
||||
"FROM seed_data "
|
||||
"WHERE json_extract(data, '$.Latitude') IS NOT NULL "
|
||||
"AND json_extract(data, '$.Longitude') IS NOT NULL"
|
||||
)
|
||||
result = {}
|
||||
for row in cur.fetchall():
|
||||
mid, lat, lon = row
|
||||
if lat and lon:
|
||||
try:
|
||||
result[mid] = {
|
||||
"lat": float(lat),
|
||||
"lon": float(lon),
|
||||
"source": "seed",
|
||||
"count": 1,
|
||||
}
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return result
|
||||
except sqlite3.OperationalError:
|
||||
return {}
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
force_update = "--force" in sys.argv
|
||||
mode = "DRY RUN (no writes)" if not apply_changes else "APPLYING changes"
|
||||
|
||||
print(f"Assets DB: {ASSET_DB}")
|
||||
print(f"Mode: {mode}")
|
||||
if force_update:
|
||||
print(" (--force: will overwrite existing GPS coordinates too)")
|
||||
print()
|
||||
|
||||
# 1. Load all GPS sources
|
||||
print("Loading GPS sources...")
|
||||
guid_to_mid = load_merged_assets()
|
||||
print(f" GUID→machine_id mappings: {len(guid_to_mid)}")
|
||||
|
||||
booking_gps = get_booking_gps(guid_to_mid)
|
||||
print(f" Booking GPS: {len(booking_gps)} machine IDs")
|
||||
|
||||
photo_gps = get_photo_gps()
|
||||
print(f" Photo EXIF GPS: {len(photo_gps)} machine IDs")
|
||||
|
||||
exif_gps = get_exif_test_gps()
|
||||
print(f" EXIF test GPS: {len(exif_gps)} machine IDs")
|
||||
|
||||
# Connect to assets DB
|
||||
conn = sqlite3.connect(ASSET_DB)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=OFF")
|
||||
|
||||
seed_gps = get_seed_gps(conn)
|
||||
print(f" Seed (Cantaloupe) GPS: {len(seed_gps)} machine IDs")
|
||||
|
||||
# 2. Get current GPS state from assets DB
|
||||
cur = conn.execute("SELECT machine_id, latitude, longitude, name, company FROM assets")
|
||||
existing = {}
|
||||
for r in cur.fetchall():
|
||||
existing[r[0]] = {"lat": r[1], "lon": r[2], "name": r[3], "company": r[4]}
|
||||
|
||||
existing_count = sum(1 for v in existing.values() if v["lat"] is not None)
|
||||
print(f"\nCurrent assets with GPS: {existing_count} / {len(existing)}")
|
||||
|
||||
# 3. Prioritize and merge sources
|
||||
all_mids = set()
|
||||
all_mids.update(booking_gps.keys())
|
||||
all_mids.update(photo_gps.keys())
|
||||
all_mids.update(exif_gps.keys())
|
||||
all_mids.update(seed_gps.keys())
|
||||
|
||||
updates = [] # (lat, lon, source, machine_id)
|
||||
skipped_existing = []
|
||||
no_match = []
|
||||
|
||||
for mid in sorted(all_mids):
|
||||
# Determine best GPS: booking > photo > exif_dev > seed
|
||||
best = None
|
||||
for src_dict, src_name in [
|
||||
(booking_gps, "booking"),
|
||||
(photo_gps, "photo"),
|
||||
(exif_gps, "exif_dev"),
|
||||
(seed_gps, "seed"),
|
||||
]:
|
||||
if mid in src_dict:
|
||||
best = src_dict[mid]
|
||||
break
|
||||
|
||||
if not best:
|
||||
continue
|
||||
|
||||
# Check if asset exists in DB
|
||||
if mid not in existing:
|
||||
no_match.append(mid)
|
||||
continue
|
||||
|
||||
existing_gps = existing[mid]
|
||||
has_gps = existing_gps["lat"] is not None and existing_gps["lon"] is not None
|
||||
|
||||
if has_gps and not force_update:
|
||||
# Code for checking distance removed — just skip
|
||||
skipped_existing.append((mid, existing_gps["lat"], existing_gps["lon"], best["lat"], best["lon"]))
|
||||
continue
|
||||
|
||||
old_label = f" ({existing_gps['lat']:.6f},{existing_gps['lon']:.6f})" if has_gps else " (no GPS)"
|
||||
updates.append((best["lat"], best["lon"], best["source"], mid))
|
||||
print(
|
||||
f" {mid:>6s} {old_label} → ({best['lat']:.6f}, {best['lon']:.6f}) [{best['source']:>7s}] "
|
||||
f"{existing_gps['name'][:40]}"
|
||||
)
|
||||
|
||||
# 4. Apply
|
||||
if apply_changes and updates:
|
||||
conn.executemany(
|
||||
"UPDATE assets SET latitude=?, longitude=?, location_source=?, updated_at=datetime('now') WHERE machine_id=?",
|
||||
[(lat, lon, src, mid) for lat, lon, src, mid in updates],
|
||||
)
|
||||
conn.commit()
|
||||
print(f"\n ✅ Applied {len(updates)} GPS updates")
|
||||
elif updates:
|
||||
print(f"\n Would update {len(updates)} assets. Run with --apply to write.")
|
||||
else:
|
||||
print("\n No updates needed.")
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total GPS sources combined: {len(all_mids)}")
|
||||
print(f" Updates to apply: {len(updates)}")
|
||||
print(f" Skipped (already has GPS): {len(skipped_existing)}")
|
||||
print(f" No matching asset in DB: {len(no_match)}")
|
||||
|
||||
# Verify final count
|
||||
if apply_changes:
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL")
|
||||
final_count = cur.fetchone()[0]
|
||||
print(f"\n Final assets with GPS: {final_count} / {len(existing)}")
|
||||
|
||||
# Print a few no-match examples
|
||||
if no_match:
|
||||
print(f"\n Sample unmapped machine IDs: {sorted(no_match)[:5]}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Geocode all asset addresses from the assets DB, cache results, and update GPS.
|
||||
|
||||
Sources (priority):
|
||||
1. Existing geocache.json
|
||||
2. Fresh Nominatim (OpenStreetMap) geocoding — rate-limited to 1/sec
|
||||
|
||||
Usage:
|
||||
python3 scripts/geocode_asset_addresses.py # dry-run
|
||||
python3 scripts/geocode_asset_addresses.py --apply # write to DB
|
||||
python3 scripts/geocode_asset_addresses.py --force # re-geocode cached too
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ASSET_DB = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
GEOCODE_CACHE = str(
|
||||
Path.home()
|
||||
/ "projects"
|
||||
/ "ms-field-service-extraction"
|
||||
/ "web"
|
||||
/ "static"
|
||||
/ "data"
|
||||
/ "geocache.json"
|
||||
)
|
||||
|
||||
USER_AGENT = "CanteenAssetTracker/1.0 (internal tool; canteen.ourpad.casa)"
|
||||
NOMINATIM_URL = "https://nominatim.openstreetmap.org/search"
|
||||
RATE_LIMIT = 1.1 # seconds between requests
|
||||
|
||||
|
||||
def load_geocache():
|
||||
"""Load existing geocache from file."""
|
||||
if Path(GEOCODE_CACHE).exists():
|
||||
with open(GEOCODE_CACHE) as f:
|
||||
return json.load(f)
|
||||
return {}
|
||||
|
||||
|
||||
def save_geocache(cache):
|
||||
"""Save geocache back to file."""
|
||||
Path(GEOCODE_CACHE).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(GEOCODE_CACHE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
|
||||
|
||||
def geocode(address):
|
||||
"""Geocode a single address via Nominatim. Returns (lat, lng) or (None, None)."""
|
||||
if not address or not address.strip():
|
||||
return None, None
|
||||
|
||||
params = urllib.parse.urlencode({
|
||||
"q": address.strip(),
|
||||
"format": "json",
|
||||
"limit": 1,
|
||||
"addressdetails": "0",
|
||||
})
|
||||
url = f"{NOMINATIM_URL}?{params}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read())
|
||||
if data and len(data) > 0:
|
||||
lat = float(data[0]["lat"])
|
||||
lng = float(data[0]["lon"])
|
||||
return lat, lng
|
||||
except Exception as e:
|
||||
print(f" ⚠ Error geocoding '{address[:60]}': {e}", file=sys.stderr)
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def make_addr_key(asset):
|
||||
"""Build a normalized address key from asset record."""
|
||||
parts = [
|
||||
asset.get("address", "") or "",
|
||||
asset.get("seed_location", "") or "",
|
||||
]
|
||||
# Build full address with city/state
|
||||
addr = ""
|
||||
if asset.get("address"):
|
||||
addr = asset["address"].strip()
|
||||
if asset.get("seed_city"):
|
||||
addr += f", {asset['seed_city'].strip()}"
|
||||
if asset.get("state"):
|
||||
addr += f", {asset['state'].strip()}"
|
||||
if asset.get("postal_code"):
|
||||
addr += f", {asset['postal_code'].strip()}"
|
||||
elif asset.get("seed_location"):
|
||||
addr = asset["seed_location"].strip()
|
||||
|
||||
return addr.strip().lower() if addr else None
|
||||
|
||||
|
||||
def build_full_address(asset):
|
||||
"""Build the cleanest full address string for geocoding.
|
||||
|
||||
Uses only the street address + city + state + zip.
|
||||
Excludes seed_location (extra venue/building names confuse Nominatim).
|
||||
"""
|
||||
addr = (asset.get("address") or "").strip()
|
||||
if not addr:
|
||||
addr = (asset.get("seed_location") or "").strip()
|
||||
|
||||
loc_parts = []
|
||||
if asset.get("seed_city"):
|
||||
loc_parts.append(asset["seed_city"].strip())
|
||||
if asset.get("state"):
|
||||
loc_parts.append(asset["state"].strip())
|
||||
if asset.get("postal_code"):
|
||||
loc_parts.append(asset["postal_code"].strip())
|
||||
|
||||
if loc_parts:
|
||||
# Add "FL" if not already present in the state field
|
||||
if "FL" not in loc_parts and "Florida" not in loc_parts:
|
||||
loc_parts.insert(0, "FL")
|
||||
addr += ", " + ", ".join(loc_parts)
|
||||
elif not addr:
|
||||
return None
|
||||
|
||||
return addr.strip().strip(",").strip()
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
force = "--force" in sys.argv
|
||||
|
||||
mode = "DRY RUN (no writes)" if not apply_changes else "APPLYING changes"
|
||||
print(f"Assets DB: {ASSET_DB}")
|
||||
print(f"Geocache: {GEOCODE_CACHE}")
|
||||
print(f"Mode: {mode}")
|
||||
if force:
|
||||
print(" (--force: re-geocode even cached addresses)")
|
||||
print()
|
||||
|
||||
# 1. Load existing cache
|
||||
cache = load_geocache()
|
||||
cached_with_coords = sum(1 for v in cache.values() if v.get("lat"))
|
||||
cached_misses = sum(1 for v in cache.values() if not v.get("lat"))
|
||||
print(f"📦 Geocache: {len(cache)} entries ({cached_with_coords} with coords, {cached_misses} misses)")
|
||||
|
||||
# 2. Get all assets without GPS but with addresses
|
||||
conn = sqlite3.connect(ASSET_DB)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("""
|
||||
SELECT machine_id, name, address, seed_location, seed_city, state, postal_code
|
||||
FROM assets
|
||||
WHERE latitude IS NULL
|
||||
AND ((address IS NOT NULL AND address != '')
|
||||
OR (seed_location IS NOT NULL AND seed_location != ''))
|
||||
ORDER BY machine_id
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
print(f"📋 Assets needing geocoding: {len(rows)}")
|
||||
|
||||
if not rows:
|
||||
print("✅ No assets need geocoding. Done.")
|
||||
return
|
||||
|
||||
# 3. Build (addr_key -> machine_ids) map for unique addresses
|
||||
addr_to_mids = {}
|
||||
addr_to_full = {}
|
||||
asset_cols = ["machine_id", "name", "address", "seed_location", "seed_city", "state", "postal_code"]
|
||||
col = dict(zip(asset_cols, range(len(asset_cols))))
|
||||
|
||||
for row in rows:
|
||||
asset = {k: row[col[k]] for k in asset_cols}
|
||||
addr_key = make_addr_key(asset)
|
||||
if not addr_key:
|
||||
continue
|
||||
if addr_key not in addr_to_mids:
|
||||
addr_to_mids[addr_key] = []
|
||||
addr_to_full[addr_key] = build_full_address(asset)
|
||||
addr_to_mids[addr_key].append(row[col["machine_id"]])
|
||||
|
||||
print(f"📍 Unique addresses to resolve: {len(addr_to_mids)}")
|
||||
|
||||
# 4. Resolve each unique address
|
||||
resolved = {} # addr_key -> (lat, lon)
|
||||
still_needed = 0
|
||||
fresh_calls = 0
|
||||
|
||||
for addr_key, mids in sorted(addr_to_mids.items()):
|
||||
# Check cache first
|
||||
cached = cache.get(addr_key, {})
|
||||
if cached and not force:
|
||||
lat, lng = cached.get("lat"), cached.get("lng")
|
||||
if lat and lng:
|
||||
resolved[addr_key] = (lat, lng)
|
||||
continue
|
||||
elif cached.get("lat") is None: # previously cached as miss
|
||||
still_needed += 1
|
||||
continue
|
||||
|
||||
# Need to geocode fresh
|
||||
full_addr = addr_to_full[addr_key]
|
||||
print(f" 🌐 Geocoding [{fresh_calls+1}]: {full_addr[:70]}...")
|
||||
lat, lng = geocode(full_addr)
|
||||
|
||||
# Save to cache
|
||||
cache[addr_key] = {"address": full_addr, "lat": lat, "lng": lng}
|
||||
save_geocache(cache)
|
||||
|
||||
if lat and lng:
|
||||
resolved[addr_key] = (lat, lng)
|
||||
print(f" ✅ ({lat:.5f}, {lng:.5f}) — affects {len(mids)} assets")
|
||||
else:
|
||||
print(f" ❌ No result — affects {len(mids)} assets")
|
||||
still_needed += 1
|
||||
|
||||
fresh_calls += 1
|
||||
if fresh_calls >= 3 and not apply_changes:
|
||||
# In dry-run mode, only do a few to show what's possible
|
||||
print(f"\n ... stopping after {fresh_calls} (dry-run). Use --apply to geocode all ~{len(addr_to_mids)} addresses.")
|
||||
break
|
||||
time.sleep(RATE_LIMIT)
|
||||
|
||||
# 5. Count updates
|
||||
total_updatable = 0
|
||||
updates = []
|
||||
for addr_key, mids in addr_to_mids.items():
|
||||
if addr_key in resolved:
|
||||
lat, lon = resolved[addr_key]
|
||||
for mid in mids:
|
||||
updates.append((lat, lon, "geocoded", mid))
|
||||
total_updatable += len(mids)
|
||||
|
||||
updates_by_source = sum(len(m) for a, m in addr_to_mids.items() if a in resolved)
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"📊 SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Total assets needing GPS: {len(rows)}")
|
||||
print(f" Unique addresses to geocode: {len(addr_to_mids)}")
|
||||
print(f" Resolved from cache/source: {len(resolved) + (cached_with_coords - sum(1 for k in resolved if k not in cache))}")
|
||||
print(f" Freshly geocoded: {fresh_calls}")
|
||||
print(f" Still unresolved: {still_needed}")
|
||||
print(f" Assets that can be updated: {updates_by_source}")
|
||||
|
||||
if apply_changes and updates:
|
||||
# Apply in batches for speed
|
||||
conn.executemany(
|
||||
"UPDATE assets SET latitude=?, longitude=?, location_source=?, updated_at=datetime('now') WHERE machine_id=?",
|
||||
[(lat, lon, src, mid) for lat, lon, src, mid in updates],
|
||||
)
|
||||
conn.commit()
|
||||
print(f"\n ✅ Applied {len(updates)} GPS updates from geocoding")
|
||||
elif updates:
|
||||
print(f"\n Run with --apply to write {len(updates)} GPS updates to DB.")
|
||||
|
||||
# Final count
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE latitude IS NOT NULL")
|
||||
final_gps = cur.fetchone()[0]
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets")
|
||||
total = cur.fetchone()[0]
|
||||
print(f"\n Final: {final_gps} / {total} assets have GPS coordinates")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate asset names to readable format and populate location fields.
|
||||
|
||||
Name format: {machine_id} - {customer} / {address} / {building_name}, Floor {floor}, Room {room}
|
||||
|
||||
Also populates missing location fields (building_name, floor, etc.)
|
||||
from seed_location, customer_name, and address data.
|
||||
|
||||
Usage:
|
||||
python3 scripts/migrate_asset_names.py # dry-run
|
||||
python3 scripts/migrate_asset_names.py --apply # write to DB
|
||||
python3 scripts/migrate_asset_names.py --force # overwrite existing location fields too
|
||||
"""
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ASSET_DB = str(Path(__file__).resolve().parent.parent / "assets.db")
|
||||
|
||||
|
||||
# ─── Building name extraction patterns ──────────────────────────────────────
|
||||
|
||||
# Disney hotel/resort name extraction from customer_name
|
||||
# "D-All Star Music GUEST" → "All Star Music"
|
||||
# "D-Port Orleans Rvsd GUEST" → "Port Orleans Riverside"
|
||||
DISNEY_RESORT_PATTERNS = [
|
||||
# "D-All Star Music GUEST" -> "All Star Music"
|
||||
(re.compile(r'^D[- ](.*?)\s+(?:GUEST|CAST|GUEST ROOM|GST)$', re.I), lambda m: m.group(1).strip()),
|
||||
# "D-All Star Music GUEST - MOV10" -> "All Star Music"
|
||||
(re.compile(r'^D[- ](.*?)\s+GUEST\s*-\s*\w+$', re.I), lambda m: m.group(1).strip()),
|
||||
# "D-CORONADO SPRINGS GUEST" -> "Coronado Springs"
|
||||
(re.compile(r'^D[- ](.*?)\s+(?:GUEST|CAST|ROOM|GUEST ROOM)$', re.I), lambda m: m.group(1).strip().title()),
|
||||
# Universal Studios / Valencia / etc.
|
||||
(re.compile(r'^Universal Studios (.+)$', re.I), lambda m: f'Universal Studios {m.group(1).strip()}'),
|
||||
]
|
||||
|
||||
# Manual overrides
|
||||
BUILDING_NAME_MAP = {
|
||||
"D-Port Orleans Rvsd GUEST": "Port Orleans Riverside",
|
||||
"D-Port Orleans Frrnc GUEST": "Port Orleans French Quarter",
|
||||
"D-CORONADO SPRINGS GUEST": "Coronado Springs",
|
||||
"D-All Star Music GUEST": "All Star Music",
|
||||
"D-All Star Movie Guest": "All Star Movies",
|
||||
"D-All Star Sports Guest": "All Star Sports",
|
||||
"D-ART OF ANIMATION Guest": "Art of Animation",
|
||||
"D-POP CENTURY GUEST": "Pop Century",
|
||||
"D-CARIBBEAN BEACH GUEST": "Caribbean Beach",
|
||||
"D-Saratoga Springs GUEST": "Saratoga Springs",
|
||||
"D-POLYNESIAN RESORT GUEST": "Polynesian Village Resort",
|
||||
"D-Animal Kngdm LodgeGuest": "Animal Kingdom Lodge",
|
||||
"D-Contemporary Guest": "Contemporary Resort",
|
||||
"D-Grand Floridian Guest": "Grand Floridian Resort",
|
||||
"D-WILDERNESS LODGE GUEST": "Wilderness Lodge",
|
||||
"D-Yacht & Beach Guest": "Yacht & Beach Club",
|
||||
"D-BoardWalk Guest": "BoardWalk Inn",
|
||||
"D-Riviera Resort Guest": "Riviera Resort",
|
||||
"D-Island Tower Polynesian": "Polynesian Village Resort - Island Tower",
|
||||
"D-DISNEY VENDING CAST": "Disney Vending",
|
||||
"D-Disney Springs CAST": "Disney Springs",
|
||||
"D-Magic Kingdom CAST": "Magic Kingdom",
|
||||
"D-Epcot CAST": "Epcot",
|
||||
"D-Hollywood Studios CAST": "Hollywood Studios",
|
||||
"D-Animal Kingdom CAST": "Animal Kingdom",
|
||||
"D-Celebration CAST": "Celebration",
|
||||
"D-WIDE WORLD SPORTS GUEST": "ESPN Wide World of Sports",
|
||||
"D-DISNEY WORLD SS CAST": "Disney World Shared Services",
|
||||
}
|
||||
|
||||
|
||||
def clean_customer_name(raw: str) -> str:
|
||||
"""Clean a customer name for display.
|
||||
|
||||
D-All Star Music GUEST → All Star Music Guest
|
||||
Home Depot #0232 Southlan → Home Depot #0232 Southlan
|
||||
"""
|
||||
if not raw:
|
||||
return ""
|
||||
name = raw
|
||||
# Remove D- prefix (Disney identifier)
|
||||
if name.startswith("D-") or name.startswith("D "):
|
||||
name = name[2:]
|
||||
# Remove GUEST/CAST/ROOM suffixes for cleaner display
|
||||
name = re.sub(r'\s+(?:GUEST|CAST|GUEST ROOM|GST|GUEST?)\s*$', '', name, flags=re.I)
|
||||
name = name.strip()
|
||||
return name
|
||||
|
||||
|
||||
def best_customer_for_name(asset: dict) -> str:
|
||||
"""Get the best customer string for the asset name."""
|
||||
c = asset.get("customer_name", "") or ""
|
||||
if c:
|
||||
return clean_customer_name(c)
|
||||
return ""
|
||||
|
||||
|
||||
def build_location_string(asset: dict) -> str:
|
||||
"""Build the location/address part for the asset name."""
|
||||
# Use address if available
|
||||
addr = (asset.get("address") or "").strip()
|
||||
# If no address, try seed_location minus customer prefix
|
||||
if not addr:
|
||||
seed = (asset.get("seed_location") or "").strip()
|
||||
if seed and " - " in seed:
|
||||
addr = seed.split(" - ", 1)[1].strip()
|
||||
elif seed:
|
||||
addr = seed
|
||||
return addr
|
||||
|
||||
|
||||
def build_building_details(asset: dict) -> str:
|
||||
"""Build building details string (building_name, floor, etc.)."""
|
||||
parts = []
|
||||
|
||||
bldg = (asset.get("building_name") or "").strip()
|
||||
floor = (asset.get("floor") or "").strip()
|
||||
room = (asset.get("room") or "").strip()
|
||||
bldg_num = (asset.get("building_number") or "").strip()
|
||||
trailer = (asset.get("trailer_number") or "").strip()
|
||||
|
||||
if bldg:
|
||||
parts.append(bldg)
|
||||
if bldg_num and (bldg not in bldg_num and bldg_num not in bldg):
|
||||
parts.append(f"Bldg {bldg_num}")
|
||||
if trailer:
|
||||
parts.append(f"Trailer {trailer}")
|
||||
if floor:
|
||||
parts.append(f"Floor {floor}")
|
||||
if room:
|
||||
parts.append(f"Room {room}")
|
||||
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def build_new_name(asset: dict) -> str:
|
||||
"""Build the new asset name."""
|
||||
mid = (asset.get("machine_id") or "").strip()
|
||||
|
||||
# Customer — cleaned
|
||||
customer = best_customer_for_name(asset)
|
||||
|
||||
# Location (street address or seed_location minus customer prefix)
|
||||
location = build_location_string(asset)
|
||||
loc_clean = ""
|
||||
if location:
|
||||
loc_clean = location.split(",")[0].strip()
|
||||
|
||||
# Building details
|
||||
building = build_building_details(asset)
|
||||
|
||||
# Avoid dupes: if building starts with loc_clean, drop the building
|
||||
if building and location:
|
||||
bldg_lower = building.lower()
|
||||
loc_lower = loc_clean.lower()
|
||||
if bldg_lower.startswith(loc_lower) or loc_lower.startswith(bldg_lower):
|
||||
# Dupes — keep whichever is more descriptive
|
||||
if len(building) > len(loc_clean):
|
||||
loc_clean = ""
|
||||
else:
|
||||
building = ""
|
||||
|
||||
# Assemble
|
||||
parts = [mid]
|
||||
|
||||
if customer:
|
||||
parts.append(f" - {customer}")
|
||||
|
||||
if location and loc_clean:
|
||||
parts.append(f" / {loc_clean}")
|
||||
|
||||
if building:
|
||||
parts.append(f" / {building}")
|
||||
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def infer_building_from_customer(customer_name: str) -> str:
|
||||
"""Try to extract a building/resort name from the customer name."""
|
||||
if not customer_name:
|
||||
return ""
|
||||
if customer_name in BUILDING_NAME_MAP:
|
||||
return BUILDING_NAME_MAP[customer_name]
|
||||
for pat, func in DISNEY_RESORT_PATTERNS:
|
||||
m = pat.search(customer_name)
|
||||
if m:
|
||||
return func(m)
|
||||
return ""
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
force = "--force" in sys.argv
|
||||
mode = "DRY RUN" if not apply_changes else "APPLYING"
|
||||
|
||||
print(f"DB: {ASSET_DB}")
|
||||
print(f"Mode: {mode}")
|
||||
if force:
|
||||
print(" (--force: overwrite existing location fields)")
|
||||
print()
|
||||
|
||||
conn = sqlite3.connect(ASSET_DB)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
|
||||
# Load all assets
|
||||
cur = conn.execute("""
|
||||
SELECT id, machine_id, name, customer_name, seed_location, address,
|
||||
building_name, floor, room, trailer_number, building_number,
|
||||
location_area, place
|
||||
FROM assets ORDER BY id
|
||||
""")
|
||||
rows = cur.fetchall()
|
||||
total = len(rows)
|
||||
print(f"Loaded {total} assets")
|
||||
|
||||
cols = ["id", "machine_id", "name", "customer_name", "seed_location", "address",
|
||||
"building_name", "floor", "room", "trailer_number", "building_number",
|
||||
"location_area", "place"]
|
||||
|
||||
stats = {
|
||||
"name_changed": 0,
|
||||
"building_name_filled": 0,
|
||||
"floor_filled": 0,
|
||||
"building_number_filled": 0,
|
||||
}
|
||||
|
||||
name_updates = []
|
||||
bldg_updates = []
|
||||
|
||||
for row in rows:
|
||||
asset = dict(zip(cols, row))
|
||||
|
||||
# ── 1. Infer building_name from customer_name if empty ──
|
||||
old_bldg = (asset["building_name"] or "").strip()
|
||||
new_bldg = old_bldg
|
||||
if not old_bldg or force:
|
||||
inferred = infer_building_from_customer(asset["customer_name"] or "")
|
||||
if inferred and inferred != old_bldg:
|
||||
new_bldg = inferred
|
||||
if not old_bldg:
|
||||
stats["building_name_filled"] += 1
|
||||
if apply_changes:
|
||||
bldg_updates.append((new_bldg, "building_name", asset["id"]))
|
||||
if mode == "DRY RUN" and not old_bldg:
|
||||
print(f" [{asset['machine_id']}] building_name '{inferred}' ← from '{asset['customer_name']}'")
|
||||
|
||||
# ── 2. Build new name ──
|
||||
old_name = (asset["name"] or "").strip()
|
||||
new_name = build_new_name(asset)
|
||||
|
||||
if new_name != old_name:
|
||||
stats["name_changed"] += 1
|
||||
name_updates.append((new_name, asset["id"]))
|
||||
if mode == "DRY RUN" and stats["name_changed"] <= 10:
|
||||
print(f" Name: {old_name[:60]}")
|
||||
print(f" → {new_name[:80]}")
|
||||
print()
|
||||
|
||||
# ── Apply ──
|
||||
if apply_changes:
|
||||
if name_updates:
|
||||
conn.executemany(
|
||||
"UPDATE assets SET name=?, updated_at=datetime('now') WHERE id=?",
|
||||
name_updates,
|
||||
)
|
||||
# Apply location updates
|
||||
for value, column, aid in bldg_updates:
|
||||
conn.execute(f"UPDATE assets SET {column}=?, updated_at=datetime('now') WHERE id=?", (value, aid))
|
||||
conn.commit()
|
||||
|
||||
# Verify
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE building_name != ''")
|
||||
bldg_count = cur.fetchone()[0]
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE floor != ''")
|
||||
floor_count = cur.fetchone()[0]
|
||||
print(f"\n ✅ Applied {len(name_updates)} name changes, {len(bldg_updates)} location updates")
|
||||
else:
|
||||
# Show some sample new names
|
||||
bldg_filled = stats["building_name_filled"]
|
||||
if name_updates:
|
||||
print(f"\n Sample new names:")
|
||||
conn2 = sqlite3.connect(ASSET_DB)
|
||||
for mid in ["95339", "95417", "98271", "90009", "95301", "68601"]:
|
||||
cur2 = conn2.execute(
|
||||
"SELECT id, machine_id, name, customer_name, address, building_name, floor FROM assets WHERE machine_id=?",
|
||||
(mid,)
|
||||
)
|
||||
for r2 in cur2.fetchall():
|
||||
a = dict(zip(cols, r2))
|
||||
print(f"\n {a['machine_id']}")
|
||||
print(f" Old: {a['name'][:70]}")
|
||||
print(f" New: {build_new_name(a)[:70]}")
|
||||
if a.get("customer_name"):
|
||||
print(f" Bldg inferred: '{infer_building_from_customer(a['customer_name'])}'")
|
||||
conn2.close()
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"SUMMARY")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Names changed: {stats['name_changed']}")
|
||||
print(f" Building names filled: {stats['building_name_filled']}")
|
||||
|
||||
if apply_changes:
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE building_name != ''")
|
||||
print(f" Total with building_name: {cur.fetchone()[0]}")
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE floor != ''")
|
||||
print(f" Total with floor: {cur.fetchone()[0]}")
|
||||
cur = conn.execute("SELECT COUNT(*) FROM assets WHERE building_name != '' OR floor != '' OR building_number != '' OR room != ''")
|
||||
print(f" Total with ANY location data: {cur.fetchone()[0]}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1039,6 +1039,7 @@ def list_assets(
|
||||
disney_filter: Optional[str] = Query(None),
|
||||
q: Optional[str] = Query(None),
|
||||
no_dex_days: Optional[int] = Query(None, ge=1),
|
||||
has_gps: Optional[bool] = Query(None),
|
||||
limit: int = Query(100, ge=1, le=2000),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
@@ -1081,6 +1082,11 @@ def list_assets(
|
||||
if no_dex_days is not None:
|
||||
conditions.append("(dex_report_date IS NULL OR REPLACE(dex_report_date, 'T', ' ') < datetime('now', '-' || ? || ' days'))")
|
||||
params.append(no_dex_days)
|
||||
if has_gps is not None:
|
||||
if has_gps:
|
||||
conditions.append("a.latitude IS NOT NULL AND a.longitude IS NOT NULL")
|
||||
else:
|
||||
conditions.append("(a.latitude IS NULL OR a.longitude IS NULL)")
|
||||
|
||||
where = " AND ".join(conditions)
|
||||
from_clause = "FROM assets a LEFT JOIN customers c ON a.customer_id = c.id"
|
||||
|
||||
+8
-6
@@ -3608,9 +3608,10 @@
|
||||
if (a.place) locParts.push(esc(a.place));
|
||||
if (locParts.length) parts.push(`<div class="ai-location-line">${locParts.join(' · ')}</div>`);
|
||||
const bldParts = [];
|
||||
if (a.building_number) bldParts.push(`Bldg ${esc(a.building_number)}`);
|
||||
if (a.building_name) bldParts.push(esc(a.building_name));
|
||||
if (a.building_number && !a.building_name?.includes(a.building_number)) bldParts.push(`Bldg ${esc(a.building_number)}`);
|
||||
if (a.trailer_number) bldParts.push(`Trailer ${esc(a.trailer_number)}`);
|
||||
if (a.floor) bldParts.push(esc(a.floor));
|
||||
if (a.floor) bldParts.push(`Floor ${esc(a.floor)}`);
|
||||
if (a.room) bldParts.push(`Room ${esc(a.room)}`);
|
||||
if (bldParts.length) parts.push(`<div class="ai-building">${bldParts.join(' · ')}</div>`);
|
||||
if (a.address) parts.push(`<div class="ai-address">📍 ${esc(a.address)}</div>`);
|
||||
@@ -3752,8 +3753,8 @@
|
||||
df('Created', formatDate(a.created_at)) +
|
||||
df('Updated', formatDate(a.updated_at));
|
||||
|
||||
// Directions
|
||||
const hasDir = a.address || a.building_name || a.building_number || a.floor || a.room || a.walking_directions || a.map_link || a.parking_location;
|
||||
// Location / Directions
|
||||
const hasDir = a.address || a.building_name || a.building_number || a.floor || a.room || a.trailer_number || a.walking_directions || a.map_link || a.parking_location;
|
||||
const dirCard = document.getElementById('detailDirections');
|
||||
const dirBtn = document.getElementById('detailDirectionsBtn');
|
||||
if (hasDir) {
|
||||
@@ -3764,6 +3765,7 @@
|
||||
df('Address', a.address),
|
||||
df('Building', a.building_name),
|
||||
df('Building #', a.building_number),
|
||||
df('Trailer', a.trailer_number),
|
||||
df('Floor', a.floor),
|
||||
df('Room', a.room),
|
||||
df('Walking Directions', a.walking_directions),
|
||||
@@ -4333,7 +4335,7 @@
|
||||
window._assetPins = [];
|
||||
|
||||
try {
|
||||
const assets = await api('/api/assets?limit=2000');
|
||||
const assets = await api('/api/assets?has_gps=true&limit=2000');
|
||||
_mapAllAssets = assets;
|
||||
buildMapFilterChips(assets);
|
||||
// Apply current filter
|
||||
@@ -4498,7 +4500,7 @@
|
||||
});
|
||||
|
||||
// Get all assets with coordinates
|
||||
const assets = await api('/api/assets?limit=2000');
|
||||
const assets = await api('/api/assets?has_gps=true&limit=2000');
|
||||
const heatData = [];
|
||||
assets.forEach(a => {
|
||||
if (a.latitude != null && a.longitude != null) {
|
||||
|
||||
Reference in New Issue
Block a user