feat: populate company/place/customer_name from MSFS extraction DB
- New script: scripts/populate_asset_company_place.py maps connect_id → account name (company/customer_name) + city (place) + address via extraction DB (msdyn_customerasset + account join) - 5,743/7,488 assets now have company + customer_name populated - 5,647/7,488 assets now have place (city) populated - Fix: SELECT COALESCE(c.name, a.customer_name) so stored customer_name isn't overridden to NULL when customer_id FK is missing - Bump SW v4→v5, version v3.1→v3.2 to force cache refresh
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Populate company, place, customer_name on assets from MSFS extraction DB.
|
||||
|
||||
Maps each asset's connect_id → msdyn_customerasset → account to get:
|
||||
- company = account name (customer/location name, e.g. "Jeremy B - 1960 Broadway")
|
||||
- place = city name (e.g. "Orlando", "Lake Buena Vista")
|
||||
- customer_name = account name (same as company, for search/filter)
|
||||
- address = line1 if empty
|
||||
|
||||
Skips assets that already have values in these fields.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ASSETS_DB = Path.home() / "projects" / "canteen-asset-tracker" / "assets.db"
|
||||
EXTRACTION_DB = (
|
||||
Path.home()
|
||||
/ "projects"
|
||||
/ "ms-field-service-extraction"
|
||||
/ "data-samples"
|
||||
/ "msfs_backup"
|
||||
/ "fieldservicecanteen.crm.dynamics.com_9fc6c50c-b097-f011-b4cb-7ced8d1b15a2_data.db"
|
||||
)
|
||||
|
||||
|
||||
def get_mapping() -> dict[str, dict]:
|
||||
"""Return {connect_id: {company, place, city, address}} from extraction DB."""
|
||||
conn = sqlite3.connect(str(EXTRACTION_DB))
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
|
||||
# We want the account info per customerasset.
|
||||
# If multiple customerassets share the same connect_id, pick the one
|
||||
# with the most recent createdon date to get freshest account link.
|
||||
rows = cur.execute("""
|
||||
SELECT
|
||||
ca.hsl_connectid,
|
||||
COALESCE(ca."msdyn_account!name", a.name, '') AS account_name,
|
||||
COALESCE(a.address1_city, '') AS city,
|
||||
COALESCE(a.address1_name, '') AS address1_name,
|
||||
COALESCE(a.address1_line1, '') AS line1,
|
||||
COALESCE(a.address1_line2, '') AS line2
|
||||
FROM msdyn_customerasset ca
|
||||
LEFT JOIN account a ON ca."msdyn_account!id" = a.accountid
|
||||
WHERE ca.hsl_connectid IS NOT NULL AND ca.hsl_connectid != ''
|
||||
ORDER BY ca.createdon DESC
|
||||
""").fetchall()
|
||||
|
||||
conn.close()
|
||||
|
||||
# Deduplicate: keep first (most recent createdon) per connect_id
|
||||
seen: set[str] = set()
|
||||
result: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
cid = r["hsl_connectid"]
|
||||
if cid in seen:
|
||||
continue
|
||||
seen.add(cid)
|
||||
|
||||
# Build address from available parts
|
||||
addr_parts = [p for p in [r["line1"], r["line2"]] if p]
|
||||
address = ", ".join(addr_parts) if addr_parts else ""
|
||||
|
||||
result[cid] = {
|
||||
"company": r["account_name"],
|
||||
"place": r["city"],
|
||||
"customer_name": r["account_name"],
|
||||
"address": address,
|
||||
}
|
||||
|
||||
print(f"📡 Loaded {len(result)} connect_id mappings from extraction DB")
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
print("🔌 Connecting to assets.db...")
|
||||
conn = sqlite3.connect(str(ASSETS_DB))
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Count assets with empty fields
|
||||
stats = conn.execute("""
|
||||
SELECT
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN company IS NULL OR company = '' THEN 1 ELSE 0 END) AS empty_company,
|
||||
SUM(CASE WHEN place IS NULL OR place = '' THEN 1 ELSE 0 END) AS empty_place,
|
||||
SUM(CASE WHEN customer_name IS NULL OR customer_name = '' THEN 1 ELSE 0 END) AS empty_customer_name,
|
||||
SUM(CASE WHEN address IS NULL OR address = '' THEN 1 ELSE 0 END) AS empty_address
|
||||
FROM assets
|
||||
""").fetchone()
|
||||
print(f"📊 Before update:")
|
||||
print(f" Total assets: {stats['total']}")
|
||||
print(f" Empty company: {stats['empty_company']}")
|
||||
print(f" Empty place: {stats['empty_place']}")
|
||||
print(f" Empty customer_name:{stats['empty_customer_name']}")
|
||||
print(f" Empty address: {stats['empty_address']}")
|
||||
|
||||
mapping = get_mapping()
|
||||
|
||||
# Fetch all assets with connect_id
|
||||
assets = conn.execute("""
|
||||
SELECT id, connect_id, company, place, customer_name, address
|
||||
FROM assets
|
||||
WHERE connect_id IS NOT NULL AND connect_id != ''
|
||||
""").fetchall()
|
||||
|
||||
updates = {"company": 0, "place": 0, "customer_name": 0, "address": 0}
|
||||
skipped_no_match = 0
|
||||
skipped_has_data = 0
|
||||
|
||||
update_sqls = {
|
||||
"company": "UPDATE assets SET company = ? WHERE id = ?",
|
||||
"place": "UPDATE assets SET place = ? WHERE id = ?",
|
||||
"customer_name": "UPDATE assets SET customer_name = ? WHERE id = ?",
|
||||
"address": "UPDATE assets SET address = ? WHERE id = ?",
|
||||
}
|
||||
|
||||
for a in assets:
|
||||
info = mapping.get(a["connect_id"])
|
||||
if not info:
|
||||
skipped_no_match += 1
|
||||
continue
|
||||
|
||||
if a["company"] and a["place"] and a["customer_name"] and a["address"]:
|
||||
skipped_has_data += 1
|
||||
continue
|
||||
|
||||
batch: list[tuple[str, int]] = [] # (sql, (value, id))
|
||||
|
||||
if (not a["company"] or not a["company"].strip()) and info["company"]:
|
||||
batch.append((update_sqls["company"], (info["company"], a["id"])))
|
||||
updates["company"] += 1
|
||||
|
||||
if (not a["place"] or not a["place"].strip()) and info["place"]:
|
||||
batch.append((update_sqls["place"], (info["place"], a["id"])))
|
||||
updates["place"] += 1
|
||||
|
||||
if (not a["customer_name"] or not a["customer_name"].strip()) and info["customer_name"]:
|
||||
batch.append((update_sqls["customer_name"], (info["customer_name"], a["id"])))
|
||||
updates["customer_name"] += 1
|
||||
|
||||
if (not a["address"] or not a["address"].strip()) and info["address"]:
|
||||
batch.append((update_sqls["address"], (info["address"], a["id"])))
|
||||
updates["address"] += 1
|
||||
|
||||
for sql, params in batch:
|
||||
conn.execute(sql, params)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(f"\n✅ Migration complete:")
|
||||
print(f" Matched & updated: {len(assets) - skipped_no_match - skipped_has_data} assets")
|
||||
print(f" No match in ext DB: {skipped_no_match}")
|
||||
print(f" Already had data: {skipped_has_data}")
|
||||
print(f" Field updates:")
|
||||
for field, count in updates.items():
|
||||
print(f" {field}: {count}")
|
||||
|
||||
# Show some samples
|
||||
conn2 = sqlite3.connect(str(ASSETS_DB))
|
||||
conn2.row_factory = sqlite3.Row
|
||||
samples = conn2.execute("""
|
||||
SELECT machine_id, company, place, customer_name, address
|
||||
FROM assets
|
||||
WHERE company != '' AND place != ''
|
||||
LIMIT 5
|
||||
""").fetchall()
|
||||
conn2.close()
|
||||
print(f"\n📋 Sample updated rows:")
|
||||
for s in samples:
|
||||
print(f" {s['machine_id']:>25} | company='{s['company']}' | place='{s['place']}'")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user