From 4f9c0d0b75a677670dd8b353e8a9f8658b055e95 Mon Sep 17 00:00:00 2001 From: Shawn Date: Fri, 29 May 2026 07:45:01 -0400 Subject: [PATCH] feat: populate company/place/customer_name from MSFS extraction DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New script: scripts/populate_asset_company_place.py maps connect_id → account name (company/customer_name) + city (place) + address via extraction DB (msdyn_customerasset + account join) - 5,743/7,488 assets now have company + customer_name populated - 5,647/7,488 assets now have place (city) populated - Fix: SELECT COALESCE(c.name, a.customer_name) so stored customer_name isn't overridden to NULL when customer_id FK is missing - Bump SW v4→v5, version v3.1→v3.2 to force cache refresh --- scripts/populate_asset_company_place.py | 177 ++++++++++++++++++++++++ server.py | 121 +++++++++++++++- static/index.html | 34 ++++- static/sw.js | 2 +- 4 files changed, 326 insertions(+), 8 deletions(-) create mode 100644 scripts/populate_asset_company_place.py diff --git a/scripts/populate_asset_company_place.py b/scripts/populate_asset_company_place.py new file mode 100644 index 0000000..88e02af --- /dev/null +++ b/scripts/populate_asset_company_place.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Populate company, place, customer_name on assets from MSFS extraction DB. + +Maps each asset's connect_id → msdyn_customerasset → account to get: + - company = account name (customer/location name, e.g. "Jeremy B - 1960 Broadway") + - place = city name (e.g. "Orlando", "Lake Buena Vista") + - customer_name = account name (same as company, for search/filter) + - address = line1 if empty + +Skips assets that already have values in these fields. +""" + +import sqlite3 +import sys +from pathlib import Path + +ASSETS_DB = Path.home() / "projects" / "canteen-asset-tracker" / "assets.db" +EXTRACTION_DB = ( + Path.home() + / "projects" + / "ms-field-service-extraction" + / "data-samples" + / "msfs_backup" + / "fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db" +) + + +def get_mapping() -> dict[str, dict]: + """Return {connect_id: {company, place, city, address}} from extraction DB.""" + conn = sqlite3.connect(str(EXTRACTION_DB)) + conn.row_factory = sqlite3.Row + cur = conn.cursor() + + # We want the account info per customerasset. + # If multiple customerassets share the same connect_id, pick the one + # with the most recent createdon date to get freshest account link. + rows = cur.execute(""" + SELECT + ca.hsl_connectid, + COALESCE(ca."msdyn_account!name", a.name, '') AS account_name, + COALESCE(a.address1_city, '') AS city, + COALESCE(a.address1_name, '') AS address1_name, + COALESCE(a.address1_line1, '') AS line1, + COALESCE(a.address1_line2, '') AS line2 + FROM msdyn_customerasset ca + LEFT JOIN account a ON ca."msdyn_account!id" = a.accountid + WHERE ca.hsl_connectid IS NOT NULL AND ca.hsl_connectid != '' + ORDER BY ca.createdon DESC + """).fetchall() + + conn.close() + + # Deduplicate: keep first (most recent createdon) per connect_id + seen: set[str] = set() + result: dict[str, dict] = {} + for r in rows: + cid = r["hsl_connectid"] + if cid in seen: + continue + seen.add(cid) + + # Build address from available parts + addr_parts = [p for p in [r["line1"], r["line2"]] if p] + address = ", ".join(addr_parts) if addr_parts else "" + + result[cid] = { + "company": r["account_name"], + "place": r["city"], + "customer_name": r["account_name"], + "address": address, + } + + print(f"📡 Loaded {len(result)} connect_id mappings from extraction DB") + return result + + +def main(): + print("🔌 Connecting to assets.db...") + conn = sqlite3.connect(str(ASSETS_DB)) + conn.row_factory = sqlite3.Row + + # Count assets with empty fields + stats = conn.execute(""" + SELECT + COUNT(*) AS total, + SUM(CASE WHEN company IS NULL OR company = '' THEN 1 ELSE 0 END) AS empty_company, + SUM(CASE WHEN place IS NULL OR place = '' THEN 1 ELSE 0 END) AS empty_place, + SUM(CASE WHEN customer_name IS NULL OR customer_name = '' THEN 1 ELSE 0 END) AS empty_customer_name, + SUM(CASE WHEN address IS NULL OR address = '' THEN 1 ELSE 0 END) AS empty_address + FROM assets + """).fetchone() + print(f"📊 Before update:") + print(f" Total assets: {stats['total']}") + print(f" Empty company: {stats['empty_company']}") + print(f" Empty place: {stats['empty_place']}") + print(f" Empty customer_name:{stats['empty_customer_name']}") + print(f" Empty address: {stats['empty_address']}") + + mapping = get_mapping() + + # Fetch all assets with connect_id + assets = conn.execute(""" + SELECT id, connect_id, company, place, customer_name, address + FROM assets + WHERE connect_id IS NOT NULL AND connect_id != '' + """).fetchall() + + updates = {"company": 0, "place": 0, "customer_name": 0, "address": 0} + skipped_no_match = 0 + skipped_has_data = 0 + + update_sqls = { + "company": "UPDATE assets SET company = ? WHERE id = ?", + "place": "UPDATE assets SET place = ? WHERE id = ?", + "customer_name": "UPDATE assets SET customer_name = ? WHERE id = ?", + "address": "UPDATE assets SET address = ? WHERE id = ?", + } + + for a in assets: + info = mapping.get(a["connect_id"]) + if not info: + skipped_no_match += 1 + continue + + if a["company"] and a["place"] and a["customer_name"] and a["address"]: + skipped_has_data += 1 + continue + + batch: list[tuple[str, int]] = [] # (sql, (value, id)) + + if (not a["company"] or not a["company"].strip()) and info["company"]: + batch.append((update_sqls["company"], (info["company"], a["id"]))) + updates["company"] += 1 + + if (not a["place"] or not a["place"].strip()) and info["place"]: + batch.append((update_sqls["place"], (info["place"], a["id"]))) + updates["place"] += 1 + + if (not a["customer_name"] or not a["customer_name"].strip()) and info["customer_name"]: + batch.append((update_sqls["customer_name"], (info["customer_name"], a["id"]))) + updates["customer_name"] += 1 + + if (not a["address"] or not a["address"].strip()) and info["address"]: + batch.append((update_sqls["address"], (info["address"], a["id"]))) + updates["address"] += 1 + + for sql, params in batch: + conn.execute(sql, params) + + conn.commit() + conn.close() + + print(f"\n✅ Migration complete:") + print(f" Matched & updated: {len(assets) - skipped_no_match - skipped_has_data} assets") + print(f" No match in ext DB: {skipped_no_match}") + print(f" Already had data: {skipped_has_data}") + print(f" Field updates:") + for field, count in updates.items(): + print(f" {field}: {count}") + + # Show some samples + conn2 = sqlite3.connect(str(ASSETS_DB)) + conn2.row_factory = sqlite3.Row + samples = conn2.execute(""" + SELECT machine_id, company, place, customer_name, address + FROM assets + WHERE company != '' AND place != '' + LIMIT 5 + """).fetchall() + conn2.close() + print(f"\n📋 Sample updated rows:") + for s in samples: + print(f" {s['machine_id']:>25} | company='{s['company']}' | place='{s['place']}'") + + +if __name__ == "__main__": + main() diff --git a/server.py b/server.py index a901ea7..977dff4 100644 --- a/server.py +++ b/server.py @@ -1040,6 +1040,7 @@ def list_assets( q: Optional[str] = Query(None), no_dex_days: Optional[int] = Query(None, ge=1), has_gps: Optional[bool] = Query(None), + branch: Optional[str] = Query(None, description="Filter by branch/territory name"), limit: int = Query(100, ge=1, le=2000), offset: int = Query(0, ge=0), ): @@ -1087,10 +1088,25 @@ def list_assets( conditions.append("a.latitude IS NOT NULL AND a.longitude IS NOT NULL") else: conditions.append("(a.latitude IS NULL OR a.longitude IS NULL)") + if branch: + cache = _build_asset_branch_cache() + # Find all connect_id prefixes that map to this branch name + matching_prefixes = [ + pfx for pfx, br in cache.items() if br == branch + ] + if matching_prefixes: + placeholders = ",".join(["?"] * len(matching_prefixes)) + conditions.append( + f"SUBSTR(a.connect_id, 1, 8) IN ({placeholders})" + ) + params.extend(matching_prefixes) + else: + # Branch name not found — return empty result + conditions.append("1=0") where = " AND ".join(conditions) from_clause = "FROM assets a LEFT JOIN customers c ON a.customer_id = c.id" - sql = f"SELECT a.*, c.name AS customer_name {from_clause}" + sql = f"SELECT a.*, COALESCE(c.name, a.customer_name) AS customer_name {from_clause}" if where: sql += f" WHERE {where}" sql += " ORDER BY a.created_at DESC LIMIT ? OFFSET ?" @@ -1121,6 +1137,44 @@ def search_by_machine_id(machine_id: str = Query(...)): return row_to_dict(row) +# ── Task 5b: GET /api/branches ───────────────────────────────────────────── + + +@app.get("/api/branches") +def list_branches(): + """Return distinct branch/territory names that assets belong to. + + Uses the connect_id prefix → branch mapping from the extraction DB. + Returns sorted list of {name, machine_count} objects. + """ + cache = _build_asset_branch_cache() + # Count how many assets per branch + conn = get_db() + prefix_counts: dict[str, int] = {} + try: + cur = conn.execute( + "SELECT SUBSTR(connect_id, 1, 8) AS prefix, COUNT(*) AS cnt " + "FROM assets WHERE connect_id IS NOT NULL AND connect_id != '' " + "GROUP BY prefix" + ) + for r in cur.fetchall(): + prefix_counts[r[0]] = r[1] + finally: + conn.close() + + # Aggregate by branch name + branch_map: dict[str, int] = {} + for prefix, machine_count in prefix_counts.items(): + branch = cache.get(prefix, prefix) + branch_map[branch] = branch_map.get(branch, 0) + machine_count + + branches = sorted( + [{"name": k, "machine_count": v} for k, v in branch_map.items()], + key=lambda b: -b["machine_count"], # type: ignore[arg-type] + ) + return {"branches": branches, "total": len(branches)} + + @app.get("/api/assets/{asset_id}") def get_asset(asset_id: int): conn = get_db() @@ -2595,6 +2649,71 @@ def _get_extraction_db() -> sqlite3.Connection | None: return conn +# ── Asset Branch Cache (connect_id prefix → branch/territory name) ──────── +_BRANCH_CACHE: dict[str, str] | None = None + + +def _build_asset_branch_cache() -> dict[str, str]: + """Build a mapping from connect_id prefix (first 8 chars) to branch name. + + Queries the extraction DB for each customer asset, linking it through + its account to work order territories (msdyn_serviceterritory!name). + Returns {connect_id_prefix: branch_name}. + """ + global _BRANCH_CACHE + if _BRANCH_CACHE is not None: + return _BRANCH_CACHE + + result: dict[str, str] = {} + conn = _get_extraction_db() + if not conn: + print("⚠️ Extraction DB not available — branch cache will be empty") + _BRANCH_CACHE = result + return result + + try: + cur = conn.cursor() + # For each customer asset (connect_id + account), find the most common + # work order territory for that account. + cur.execute(""" + SELECT + SUBSTR(ca.hsl_connectid, 1, 8) AS prefix, + wo."msdyn_serviceterritory!name" AS territory, + COUNT(*) AS cnt + FROM msdyn_customerasset ca + LEFT JOIN account a ON ca."msdyn_account!id" = a.accountid + LEFT JOIN msdyn_workorder wo ON a.accountid = wo."msdyn_serviceaccount!id" + WHERE ca.hsl_connectid IS NOT NULL AND ca.hsl_connectid != '' + AND wo."msdyn_serviceterritory!name" IS NOT NULL + AND wo."msdyn_serviceterritory!name" != '' + GROUP BY prefix, wo."msdyn_serviceterritory!name" + ORDER BY cnt DESC + """) + # Take the most frequent territory per prefix + prefix_map: dict[str, list[tuple[str, int]]] = {} + for r in cur.fetchall(): + prefix = r[0] + territory = r[1] + cnt = r[2] + if prefix not in prefix_map: + prefix_map[prefix] = [] + prefix_map[prefix].append((territory, cnt)) + + for prefix, territories in prefix_map.items(): + # Sort by count descending, pick the top one + territories.sort(key=lambda x: -x[1]) + result[prefix] = territories[0][0] + + print(f"📡 Built branch cache: {len(result)} connect_id prefixes mapped") + except Exception as e: + print(f"⚠️ Failed to build branch cache: {e}") + finally: + conn.close() + + _BRANCH_CACHE = result + return result + + # ── TSP Solver ─────────────────────────────────────────────────────────── diff --git a/static/index.html b/static/index.html index c247933..3594a86 100644 --- a/static/index.html +++ b/static/index.html @@ -1091,7 +1091,7 @@
-
Canteen Asset Tracker v3.0
+
Canteen Asset Tracker v3.2
Esc to close · Ctrl+/ to search · ❓ for help
@@ -1228,6 +1228,14 @@
+
+
+
Branch
+ +
+
Data Health
@@ -3323,7 +3331,7 @@ // ── Filter state ────────────────────────────────────────────────────── let assetFilters = { category: null, status: null, make: null, disney_park: null, disney_filter: null, q: '', - customer_id: null, location_id: null, assigned_to: null }; + customer_id: null, location_id: null, assigned_to: null, branch: null }; let noDexFilterActive = false; let assetOffset = 0; const ASSET_PAGE_SIZE = 2000; @@ -3372,6 +3380,11 @@ assnSel.innerHTML = '' + users.filter(function(u) { return u.role === 'technician' || u.role === 'admin'; }) .map(function(u) { return ''; }).join(''); + // Load branches + var branches = await api('/api/branches'); + var branchSel = document.getElementById('filterBranch'); + branchSel.innerHTML = '' + + branches.branches.map(function(b) { return ''; }).join(''); } catch (e) { showToast('Failed to load filter options', true); } finally { @@ -3406,6 +3419,7 @@ var locEl = document.getElementById('filterLocation'); var assnEl = document.getElementById('filterAssignedTo'); + var branchEl = document.getElementById('filterBranch'); assetFilters.category = catEl.value || null; assetFilters.make = makeEl.value || null; @@ -3422,6 +3436,7 @@ assetFilters.customer_id = custEl.value || null; assetFilters.assigned_to = assnEl.value || null; + assetFilters.branch = branchEl.value || null; var prevCust = locEl.dataset.customerId; if (custEl.value !== prevCust) { @@ -3461,6 +3476,7 @@ if (assetFilters.customer_id) count++; if (assetFilters.location_id) count++; if (assetFilters.assigned_to) count++; + if (assetFilters.branch) count++; if (noDexFilterActive) count++; if (assetFilters.q) count++; return count; @@ -3492,10 +3508,11 @@ if (assetFilters.location_id) { var locSel = document.getElementById('filterLocation'); var txt = locSel.options[locSel.selectedIndex] ? locSel.options[locSel.selectedIndex].text : 'Location'; - tags.push({key:'location_id', label:'\uD83D\uDCCD ' + txt}); + tags.push({key:'location_id', label:'📍 ' + txt}); } - if (assetFilters.assigned_to) tags.push({key:'assigned_to', label:'\uD83D\uDC64 ' + assetFilters.assigned_to}); - if (noDexFilterActive) tags.push({key:'no_dex_days', label:'\u26a0\ufe0f No DEX (5d)'}); + if (assetFilters.assigned_to) tags.push({key:'assigned_to', label:'👤 ' + assetFilters.assigned_to}); + if (assetFilters.branch) tags.push({key:'branch', label:'🏢 ' + assetFilters.branch}); + if (noDexFilterActive) tags.push({key:'no_dex_days', label:'⚠️ No DEX (5d)'}); el.innerHTML = tags.map(function(t) { return '' + esc(t.label) + ' \u2715'; @@ -3538,6 +3555,9 @@ } else if (key === 'assigned_to') { document.getElementById('filterAssignedTo').value = ''; assetFilters.assigned_to = null; + } else if (key === 'branch') { + document.getElementById('filterBranch').value = ''; + assetFilters.branch = null; } else if (key === 'no_dex_days') { noDexFilterActive = false; var btn = document.getElementById('filterNoDex'); @@ -3550,7 +3570,7 @@ function clearAllFilters() { assetFilters = { category: null, status: null, make: null, disney_park: null, disney_filter: null, q: '', - customer_id: null, location_id: null, assigned_to: null }; + customer_id: null, location_id: null, assigned_to: null, branch: null }; document.getElementById('assetSearch').value = ''; document.getElementById('clearSearch').style.display = 'none'; document.getElementById('filterCategory').value = ''; @@ -3562,6 +3582,7 @@ document.getElementById('filterLocation').disabled = true; document.getElementById('filterAssignedTo').value = ''; + document.getElementById('filterBranch').value = ''; noDexFilterActive = false; var noDexBtn = document.getElementById('filterNoDex'); if (noDexBtn) noDexBtn.classList.remove('active'); @@ -3597,6 +3618,7 @@ if (assetFilters.customer_id) params.set('customer_id', assetFilters.customer_id); if (assetFilters.location_id) params.set('location_id', assetFilters.location_id); if (assetFilters.assigned_to) params.set('assigned_to', assetFilters.assigned_to); + if (assetFilters.branch) params.set('branch', assetFilters.branch); if (noDexFilterActive) params.set('no_dex_days', '5'); params.set('limit', String(ASSET_PAGE_SIZE)); params.set('offset', String(assetOffset)); diff --git a/static/sw.js b/static/sw.js index 6deb890..d90a539 100644 --- a/static/sw.js +++ b/static/sw.js @@ -3,7 +3,7 @@ // Caches app shell + CDN deps. Network-first for API, cache fallback for static. // ═══════════════════════════════════════════════════════════════════════════ -const CACHE_NAME = 'canteen-v3'; +const CACHE_NAME = 'canteen-v5'; // App shell — core resources needed to boot the PWA const APP_SHELL = [