feat: populate company/place/customer_name from MSFS extraction DB
- New script: scripts/populate_asset_company_place.py maps connect_id → account name (company/customer_name) + city (place) + address via extraction DB (msdyn_customerasset + account join) - 5,743/7,488 assets now have company + customer_name populated - 5,647/7,488 assets now have place (city) populated - Fix: SELECT COALESCE(c.name, a.customer_name) so stored customer_name isn't overridden to NULL when customer_id FK is missing - Bump SW v4→v5, version v3.1→v3.2 to force cache refresh
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Populate company, place, customer_name on assets from MSFS extraction DB.
|
||||
|
||||
Maps each asset's connect_id → msdyn_customerasset → account to get:
|
||||
- company = account name (customer/location name, e.g. "Jeremy B - 1960 Broadway")
|
||||
- place = city name (e.g. "Orlando", "Lake Buena Vista")
|
||||
- customer_name = account name (same as company, for search/filter)
|
||||
- address = line1 if empty
|
||||
|
||||
Skips assets that already have values in these fields.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ASSETS_DB = Path.home() / "projects" / "canteen-asset-tracker" / "assets.db"
|
||||
EXTRACTION_DB = (
|
||||
Path.home()
|
||||
/ "projects"
|
||||
/ "ms-field-service-extraction"
|
||||
/ "data-samples"
|
||||
/ "msfs_backup"
|
||||
/ "fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db"
|
||||
)
|
||||
|
||||
|
||||
def get_mapping() -> dict[str, dict]:
|
||||
"""Return {connect_id: {company, place, city, address}} from extraction DB."""
|
||||
conn = sqlite3.connect(str(EXTRACTION_DB))
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
|
||||
# We want the account info per customerasset.
|
||||
# If multiple customerassets share the same connect_id, pick the one
|
||||
# with the most recent createdon date to get freshest account link.
|
||||
rows = cur.execute("""
|
||||
SELECT
|
||||
ca.hsl_connectid,
|
||||
COALESCE(ca."msdyn_account!name", a.name, '') AS account_name,
|
||||
COALESCE(a.address1_city, '') AS city,
|
||||
COALESCE(a.address1_name, '') AS address1_name,
|
||||
COALESCE(a.address1_line1, '') AS line1,
|
||||
COALESCE(a.address1_line2, '') AS line2
|
||||
FROM msdyn_customerasset ca
|
||||
LEFT JOIN account a ON ca."msdyn_account!id" = a.accountid
|
||||
WHERE ca.hsl_connectid IS NOT NULL AND ca.hsl_connectid != ''
|
||||
ORDER BY ca.createdon DESC
|
||||
""").fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
# Deduplicate: keep first (most recent createdon) per connect_id
|
||||
seen: set[str] = set()
|
||||
result: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
cid = r["hsl_connectid"]
|
||||
if cid in seen:
|
||||
continue
|
||||
seen.add(cid)
|
||||
|
||||
# Build address from available parts
|
||||
addr_parts = [p for p in [r["line1"], r["line2"]] if p]
|
||||
address = ", ".join(addr_parts) if addr_parts else ""
|
||||
|
||||
result[cid] = {
|
||||
"company": r["account_name"],
|
||||
"place": r["city"],
|
||||
"customer_name": r["account_name"],
|
||||
"address": address,
|
||||
}
|
||||
|
||||
print(f"📡 Loaded {len(result)} connect_id mappings from extraction DB")
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
print("🔌 Connecting to assets.db...")
|
||||
conn = sqlite3.connect(str(ASSETS_DB))
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Count assets with empty fields
|
||||
stats = conn.execute("""
|
||||
SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN company IS NULL OR company = '' THEN 1 ELSE 0 END) AS empty_company,
|
||||
SUM(CASE WHEN place IS NULL OR place = '' THEN 1 ELSE 0 END) AS empty_place,
|
||||
SUM(CASE WHEN customer_name IS NULL OR customer_name = '' THEN 1 ELSE 0 END) AS empty_customer_name,
|
||||
SUM(CASE WHEN address IS NULL OR address = '' THEN 1 ELSE 0 END) AS empty_address
|
||||
FROM assets
|
||||
""").fetchone()
|
||||
print(f"📊 Before update:")
|
||||
print(f" Total assets: {stats['total']}")
|
||||
print(f" Empty company: {stats['empty_company']}")
|
||||
print(f" Empty place: {stats['empty_place']}")
|
||||
print(f" Empty customer_name:{stats['empty_customer_name']}")
|
||||
print(f" Empty address: {stats['empty_address']}")
|
||||
|
||||
mapping = get_mapping()
|
||||
|
||||
# Fetch all assets with connect_id
|
||||
assets = conn.execute("""
|
||||
SELECT id, connect_id, company, place, customer_name, address
|
||||
FROM assets
|
||||
WHERE connect_id IS NOT NULL AND connect_id != ''
|
||||
""").fetchall()
|
||||
|
||||
updates = {"company": 0, "place": 0, "customer_name": 0, "address": 0}
|
||||
skipped_no_match = 0
|
||||
skipped_has_data = 0
|
||||
|
||||
update_sqls = {
|
||||
"company": "UPDATE assets SET company = ? WHERE id = ?",
|
||||
"place": "UPDATE assets SET place = ? WHERE id = ?",
|
||||
"customer_name": "UPDATE assets SET customer_name = ? WHERE id = ?",
|
||||
"address": "UPDATE assets SET address = ? WHERE id = ?",
|
||||
}
|
||||
|
||||
for a in assets:
|
||||
info = mapping.get(a["connect_id"])
|
||||
if not info:
|
||||
skipped_no_match += 1
|
||||
continue
|
||||
|
||||
if a["company"] and a["place"] and a["customer_name"] and a["address"]:
|
||||
skipped_has_data += 1
|
||||
continue
|
||||
|
||||
batch: list[tuple[str, int]] = [] # (sql, (value, id))
|
||||
|
||||
if (not a["company"] or not a["company"].strip()) and info["company"]:
|
||||
batch.append((update_sqls["company"], (info["company"], a["id"])))
|
||||
updates["company"] += 1
|
||||
|
||||
if (not a["place"] or not a["place"].strip()) and info["place"]:
|
||||
batch.append((update_sqls["place"], (info["place"], a["id"])))
|
||||
updates["place"] += 1
|
||||
|
||||
if (not a["customer_name"] or not a["customer_name"].strip()) and info["customer_name"]:
|
||||
batch.append((update_sqls["customer_name"], (info["customer_name"], a["id"])))
|
||||
updates["customer_name"] += 1
|
||||
|
||||
if (not a["address"] or not a["address"].strip()) and info["address"]:
|
||||
batch.append((update_sqls["address"], (info["address"], a["id"])))
|
||||
updates["address"] += 1
|
||||
|
||||
for sql, params in batch:
|
||||
conn.execute(sql, params)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(f"\n✅ Migration complete:")
|
||||
print(f" Matched & updated: {len(assets) - skipped_no_match - skipped_has_data} assets")
|
||||
print(f" No match in ext DB: {skipped_no_match}")
|
||||
print(f" Already had data: {skipped_has_data}")
|
||||
print(f" Field updates:")
|
||||
for field, count in updates.items():
|
||||
print(f" {field}: {count}")
|
||||
|
||||
# Show some samples
|
||||
conn2 = sqlite3.connect(str(ASSETS_DB))
|
||||
conn2.row_factory = sqlite3.Row
|
||||
samples = conn2.execute("""
|
||||
SELECT machine_id, company, place, customer_name, address
|
||||
FROM assets
|
||||
WHERE company != '' AND place != ''
|
||||
LIMIT 5
|
||||
""").fetchall()
|
||||
conn2.close()
|
||||
print(f"\n📋 Sample updated rows:")
|
||||
for s in samples:
|
||||
print(f" {s['machine_id']:>25} | company='{s['company']}' | place='{s['place']}'")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1040,6 +1040,7 @@ def list_assets(
|
||||
q: Optional[str] = Query(None),
|
||||
no_dex_days: Optional[int] = Query(None, ge=1),
|
||||
has_gps: Optional[bool] = Query(None),
|
||||
branch: Optional[str] = Query(None, description="Filter by branch/territory name"),
|
||||
limit: int = Query(100, ge=1, le=2000),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
@@ -1087,10 +1088,25 @@ def list_assets(
|
||||
conditions.append("a.latitude IS NOT NULL AND a.longitude IS NOT NULL")
|
||||
else:
|
||||
conditions.append("(a.latitude IS NULL OR a.longitude IS NULL)")
|
||||
if branch:
|
||||
cache = _build_asset_branch_cache()
|
||||
# Find all connect_id prefixes that map to this branch name
|
||||
matching_prefixes = [
|
||||
pfx for pfx, br in cache.items() if br == branch
|
||||
]
|
||||
if matching_prefixes:
|
||||
placeholders = ",".join(["?"] * len(matching_prefixes))
|
||||
conditions.append(
|
||||
f"SUBSTR(a.connect_id, 1, 8) IN ({placeholders})"
|
||||
)
|
||||
params.extend(matching_prefixes)
|
||||
else:
|
||||
# Branch name not found — return empty result
|
||||
conditions.append("1=0")
|
||||
|
||||
where = " AND ".join(conditions)
|
||||
from_clause = "FROM assets a LEFT JOIN customers c ON a.customer_id = c.id"
|
||||
sql = f"SELECT a.*, c.name AS customer_name {from_clause}"
|
||||
sql = f"SELECT a.*, COALESCE(c.name, a.customer_name) AS customer_name {from_clause}"
|
||||
if where:
|
||||
sql += f" WHERE {where}"
|
||||
sql += " ORDER BY a.created_at DESC LIMIT ? OFFSET ?"
|
||||
@@ -1121,6 +1137,44 @@ def search_by_machine_id(machine_id: str = Query(...)):
|
||||
return row_to_dict(row)
|
||||
|
||||
|
||||
# ── Task 5b: GET /api/branches ─────────────────────────────────────────────
|
||||
|
||||
|
||||
@app.get("/api/branches")
|
||||
def list_branches():
|
||||
"""Return distinct branch/territory names that assets belong to.
|
||||
|
||||
Uses the connect_id prefix → branch mapping from the extraction DB.
|
||||
Returns sorted list of {name, machine_count} objects.
|
||||
"""
|
||||
cache = _build_asset_branch_cache()
|
||||
# Count how many assets per branch
|
||||
conn = get_db()
|
||||
prefix_counts: dict[str, int] = {}
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"SELECT SUBSTR(connect_id, 1, 8) AS prefix, COUNT(*) AS cnt "
|
||||
"FROM assets WHERE connect_id IS NOT NULL AND connect_id != '' "
|
||||
"GROUP BY prefix"
|
||||
)
|
||||
for r in cur.fetchall():
|
||||
prefix_counts[r[0]] = r[1]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# Aggregate by branch name
|
||||
branch_map: dict[str, int] = {}
|
||||
for prefix, machine_count in prefix_counts.items():
|
||||
branch = cache.get(prefix, prefix)
|
||||
branch_map[branch] = branch_map.get(branch, 0) + machine_count
|
||||
|
||||
branches = sorted(
|
||||
[{"name": k, "machine_count": v} for k, v in branch_map.items()],
|
||||
key=lambda b: -b["machine_count"], # type: ignore[arg-type]
|
||||
)
|
||||
return {"branches": branches, "total": len(branches)}
|
||||
|
||||
|
||||
@app.get("/api/assets/{asset_id}")
|
||||
def get_asset(asset_id: int):
|
||||
conn = get_db()
|
||||
@@ -2595,6 +2649,71 @@ def _get_extraction_db() -> sqlite3.Connection | None:
|
||||
return conn
|
||||
|
||||
|
||||
# ── Asset Branch Cache (connect_id prefix → branch/territory name) ────────
|
||||
_BRANCH_CACHE: dict[str, str] | None = None
|
||||
|
||||
|
||||
def _build_asset_branch_cache() -> dict[str, str]:
|
||||
"""Build a mapping from connect_id prefix (first 8 chars) to branch name.
|
||||
|
||||
Queries the extraction DB for each customer asset, linking it through
|
||||
its account to work order territories (msdyn_serviceterritory!name).
|
||||
Returns {connect_id_prefix: branch_name}.
|
||||
"""
|
||||
global _BRANCH_CACHE
|
||||
if _BRANCH_CACHE is not None:
|
||||
return _BRANCH_CACHE
|
||||
|
||||
result: dict[str, str] = {}
|
||||
conn = _get_extraction_db()
|
||||
if not conn:
|
||||
print("⚠️ Extraction DB not available — branch cache will be empty")
|
||||
_BRANCH_CACHE = result
|
||||
return result
|
||||
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
# For each customer asset (connect_id + account), find the most common
|
||||
# work order territory for that account.
|
||||
cur.execute("""
|
||||
SELECT
|
||||
SUBSTR(ca.hsl_connectid, 1, 8) AS prefix,
|
||||
wo."msdyn_serviceterritory!name" AS territory,
|
||||
COUNT(*) AS cnt
|
||||
FROM msdyn_customerasset ca
|
||||
LEFT JOIN account a ON ca."msdyn_account!id" = a.accountid
|
||||
LEFT JOIN msdyn_workorder wo ON a.accountid = wo."msdyn_serviceaccount!id"
|
||||
WHERE ca.hsl_connectid IS NOT NULL AND ca.hsl_connectid != ''
|
||||
AND wo."msdyn_serviceterritory!name" IS NOT NULL
|
||||
AND wo."msdyn_serviceterritory!name" != ''
|
||||
GROUP BY prefix, wo."msdyn_serviceterritory!name"
|
||||
ORDER BY cnt DESC
|
||||
""")
|
||||
# Take the most frequent territory per prefix
|
||||
prefix_map: dict[str, list[tuple[str, int]]] = {}
|
||||
for r in cur.fetchall():
|
||||
prefix = r[0]
|
||||
territory = r[1]
|
||||
cnt = r[2]
|
||||
if prefix not in prefix_map:
|
||||
prefix_map[prefix] = []
|
||||
prefix_map[prefix].append((territory, cnt))
|
||||
|
||||
for prefix, territories in prefix_map.items():
|
||||
# Sort by count descending, pick the top one
|
||||
territories.sort(key=lambda x: -x[1])
|
||||
result[prefix] = territories[0][0]
|
||||
|
||||
print(f"📡 Built branch cache: {len(result)} connect_id prefixes mapped")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Failed to build branch cache: {e}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
_BRANCH_CACHE = result
|
||||
return result
|
||||
|
||||
|
||||
# ── TSP Solver ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
+28
-6
@@ -1091,7 +1091,7 @@
|
||||
</button>
|
||||
</nav>
|
||||
<div class="drawer-footer">
|
||||
<div style="margin-bottom:6px;">Canteen Asset Tracker v3.0</div>
|
||||
<div style="margin-bottom:6px;">Canteen Asset Tracker v3.2</div>
|
||||
<div style="font-size:11px;line-height:1.5;margin-bottom:6px;">
|
||||
<kbd>Esc</kbd> to close · <kbd>Ctrl</kbd>+<kbd>/</kbd> to search · ❓ for help
|
||||
</div>
|
||||
@@ -1228,6 +1228,14 @@
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="fp-row">
|
||||
<div style="flex:1">
|
||||
<div class="fp-label">Branch</div>
|
||||
<select id="filterBranch" class="input-field" onchange="onFilterChange()">
|
||||
<option value="">All branches</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="fp-row">
|
||||
<div style="flex:1">
|
||||
<div class="fp-label">Data Health</div>
|
||||
@@ -3323,7 +3331,7 @@
|
||||
|
||||
// ── Filter state ──────────────────────────────────────────────────────
|
||||
let assetFilters = { category: null, status: null, make: null, disney_park: null, disney_filter: null, q: '',
|
||||
customer_id: null, location_id: null, assigned_to: null };
|
||||
customer_id: null, location_id: null, assigned_to: null, branch: null };
|
||||
let noDexFilterActive = false;
|
||||
let assetOffset = 0;
|
||||
const ASSET_PAGE_SIZE = 2000;
|
||||
@@ -3372,6 +3380,11 @@
|
||||
assnSel.innerHTML = '<option value="">Anyone</option>' +
|
||||
users.filter(function(u) { return u.role === 'technician' || u.role === 'admin'; })
|
||||
.map(function(u) { return '<option value="' + esc(u.username) + '">' + esc(u.username) + '</option>'; }).join('');
|
||||
// Load branches
|
||||
var branches = await api('/api/branches');
|
||||
var branchSel = document.getElementById('filterBranch');
|
||||
branchSel.innerHTML = '<option value="">All branches</option>' +
|
||||
branches.branches.map(function(b) { return '<option value="' + esc(b.name) + '">' + esc(b.name) + ' (' + b.machine_count + ')</option>'; }).join('');
|
||||
} catch (e) {
|
||||
showToast('Failed to load filter options', true);
|
||||
} finally {
|
||||
@@ -3406,6 +3419,7 @@
|
||||
var locEl = document.getElementById('filterLocation');
|
||||
|
||||
var assnEl = document.getElementById('filterAssignedTo');
|
||||
var branchEl = document.getElementById('filterBranch');
|
||||
|
||||
assetFilters.category = catEl.value || null;
|
||||
assetFilters.make = makeEl.value || null;
|
||||
@@ -3422,6 +3436,7 @@
|
||||
assetFilters.customer_id = custEl.value || null;
|
||||
|
||||
assetFilters.assigned_to = assnEl.value || null;
|
||||
assetFilters.branch = branchEl.value || null;
|
||||
|
||||
var prevCust = locEl.dataset.customerId;
|
||||
if (custEl.value !== prevCust) {
|
||||
@@ -3461,6 +3476,7 @@
|
||||
if (assetFilters.customer_id) count++;
|
||||
if (assetFilters.location_id) count++;
|
||||
if (assetFilters.assigned_to) count++;
|
||||
if (assetFilters.branch) count++;
|
||||
if (noDexFilterActive) count++;
|
||||
if (assetFilters.q) count++;
|
||||
return count;
|
||||
@@ -3492,10 +3508,11 @@
|
||||
if (assetFilters.location_id) {
|
||||
var locSel = document.getElementById('filterLocation');
|
||||
var txt = locSel.options[locSel.selectedIndex] ? locSel.options[locSel.selectedIndex].text : 'Location';
|
||||
tags.push({key:'location_id', label:'\uD83D\uDCCD ' + txt});
|
||||
tags.push({key:'location_id', label:'📍 ' + txt});
|
||||
}
|
||||
if (assetFilters.assigned_to) tags.push({key:'assigned_to', label:'\uD83D\uDC64 ' + assetFilters.assigned_to});
|
||||
if (noDexFilterActive) tags.push({key:'no_dex_days', label:'\u26a0\ufe0f No DEX (5d)'});
|
||||
if (assetFilters.assigned_to) tags.push({key:'assigned_to', label:'👤 ' + assetFilters.assigned_to});
|
||||
if (assetFilters.branch) tags.push({key:'branch', label:'🏢 ' + assetFilters.branch});
|
||||
if (noDexFilterActive) tags.push({key:'no_dex_days', label:'⚠️ No DEX (5d)'});
|
||||
|
||||
el.innerHTML = tags.map(function(t) {
|
||||
return '<span class="filter-tag">' + esc(t.label) + ' <span class="ft-remove" onclick="removeFilter(\'' + t.key + '\')">\u2715</span></span>';
|
||||
@@ -3538,6 +3555,9 @@
|
||||
} else if (key === 'assigned_to') {
|
||||
document.getElementById('filterAssignedTo').value = '';
|
||||
assetFilters.assigned_to = null;
|
||||
} else if (key === 'branch') {
|
||||
document.getElementById('filterBranch').value = '';
|
||||
assetFilters.branch = null;
|
||||
} else if (key === 'no_dex_days') {
|
||||
noDexFilterActive = false;
|
||||
var btn = document.getElementById('filterNoDex');
|
||||
@@ -3550,7 +3570,7 @@
|
||||
|
||||
function clearAllFilters() {
|
||||
assetFilters = { category: null, status: null, make: null, disney_park: null, disney_filter: null, q: '',
|
||||
customer_id: null, location_id: null, assigned_to: null };
|
||||
customer_id: null, location_id: null, assigned_to: null, branch: null };
|
||||
document.getElementById('assetSearch').value = '';
|
||||
document.getElementById('clearSearch').style.display = 'none';
|
||||
document.getElementById('filterCategory').value = '';
|
||||
@@ -3562,6 +3582,7 @@
|
||||
document.getElementById('filterLocation').disabled = true;
|
||||
|
||||
document.getElementById('filterAssignedTo').value = '';
|
||||
document.getElementById('filterBranch').value = '';
|
||||
noDexFilterActive = false;
|
||||
var noDexBtn = document.getElementById('filterNoDex');
|
||||
if (noDexBtn) noDexBtn.classList.remove('active');
|
||||
@@ -3597,6 +3618,7 @@
|
||||
if (assetFilters.customer_id) params.set('customer_id', assetFilters.customer_id);
|
||||
if (assetFilters.location_id) params.set('location_id', assetFilters.location_id);
|
||||
if (assetFilters.assigned_to) params.set('assigned_to', assetFilters.assigned_to);
|
||||
if (assetFilters.branch) params.set('branch', assetFilters.branch);
|
||||
if (noDexFilterActive) params.set('no_dex_days', '5');
|
||||
params.set('limit', String(ASSET_PAGE_SIZE));
|
||||
params.set('offset', String(assetOffset));
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
// Caches app shell + CDN deps. Network-first for API, cache fallback for static.
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const CACHE_NAME = 'canteen-v3';
|
||||
const CACHE_NAME = 'canteen-v5';
|
||||
|
||||
// App shell — core resources needed to boot the PWA
|
||||
const APP_SHELL = [
|
||||
|
||||
Reference in New Issue
Block a user