cantaloupe_sync: fix column mapping heuristic, add comprehensive tests

- Fix greedy matching where 'name' keyword matched 'Location Name' header
- Use word-subset matching instead of substring matching
- Normalize keywords (replace _ and # with spaces) to match header normalization
- Reorder heuristics: multi-word specific patterns before single-word generics
- Add 10 unit tests for mapping, diff, and Excel parsing
- Add 23 integration tests for all sync API endpoints
- All 33 tests pass
This commit is contained in:
Leo
2026-05-21 19:16:53 -04:00
parent d2c76aea7f
commit f7ffa0e6cc
3 changed files with 487 additions and 25 deletions
+54 -25
View File
@@ -98,37 +98,59 @@ ASSET_COLUMNS = [
COLUMN_HEURISTICS = [
# machine_id is the primary key for upserts
(["asset id", "assetid", "machine id", "machineid", "machine_id", "equipment id", "unit id", "asset number"], "machine_id"),
(["serial", "serial number", "serialnumber", "serial_no", "s/n"], "serial_number"),
(["name", "asset name", "equipment name", "description"], "name"),
(["description", "notes", "detail", "comments"], "description"),
# Multi-word / specific heuristics first to avoid greedy single-word matches
(["location name", "location_name", "site name"], "_location_name"),
(["building name", "building_name", "bldg name"], "building_name"),
(["building number", "building_number", "building_no", "bldg number", "bldg #", "building #"], "building_number"),
(["trailer number", "trailer_number", "trailer_no", "trailer #", "mobile unit"], "trailer_number"),
(["walking directions", "walking_directions", "how to get there"], "walking_directions"),
(["map link", "map_link", "google maps", "map url", "location url"], "map_link"),
(["parking location", "parking_location", "parking spot"], "parking_location"),
(["street address", "location address", "site address"], "address"),
(["asset name", "equipment name"], "name"),
(["model number", "model_no", "model #"], "model"),
(["serial number", "serialnumber", "serial_no", "s/n"], "serial_number"),
(["room number", "room #", "suite"], "room"),
(["customer name", "client name", "account name", "company"], "_customer_name"),
(["photo path", "photo_path", "image path"], "photo_path"),
(["route number", "route #", "delivery route"], "_route"),
(["sales rep", "salesperson", "account manager"], "_sales_rep"),
(["qr code", "tag number", "tag #"], "_barcode"),
(["geofence radius", "geofence_radius", "geo radius"], "geofence_radius_meters"),
(["gps lat", "lat"], "latitude"),
(["gps lng", "gps long", "long", "lng", "lon"], "longitude"),
# Single-word/generic heuristics (last, after specific multi-word ones)
(["serial", "s/n"], "serial_number"),
(["category", "type", "asset type", "equipment type", "class"], "category"),
(["status", "state", "condition"], "status"),
(["make", "manufacturer", "brand", "vendor"], "make"),
(["model", "model number", "model_no", "model #"], "model"),
(["address", "street", "street address", "location address", "site address"], "address"),
(["building name", "building_name", "bldg name", "site name"], "building_name"),
(["building number", "building_number", "building_no", "bldg number", "bldg #", "building #"], "building_number"),
(["address", "street"], "address"),
(["floor", "level", "story"], "floor"),
(["room", "room number", "room #", "suite"], "room"),
(["trailer", "trailer number", "trailer_no", "trailer #", "mobile unit"], "trailer_number"),
(["walking directions", "walking_directions", "directions", "how to get there"], "walking_directions"),
(["map link", "map_link", "google maps", "map url", "location url"], "map_link"),
(["parking", "parking location", "parking_location", "parking spot"], "parking_location"),
(["latitude", "lat", "gps lat"], "latitude"),
(["longitude", "long", "lng", "lon", "gps lng", "gps long"], "longitude"),
(["geofence", "geo radius", "radius", "geofence radius", "geofence_radius"], "geofence_radius_meters"),
(["customer", "client", "account", "company", "customer name"], "_customer_name"),
(["location name", "location_name", "site", "site name", "location"], "_location_name"),
(["photo", "photo path", "photo_path", "image", "picture"], "photo_path"),
(["route", "route number", "route #", "delivery route"], "_route"),
(["sales", "sales rep", "salesperson", "account manager"], "_sales_rep"),
(["barcode", "qr code", "tag", "tag number", "tag #"], "_barcode"),
(["latitude", "gps"], "latitude"),
(["longitude", "geo"], "longitude"),
(["geofence", "radius"], "geofence_radius_meters"),
(["customer", "client", "account"], "_customer_name"),
(["location", "site"], "_location_name"),
(["name"], "name"),
(["description", "notes", "detail", "comments"], "description"),
(["model"], "model"),
(["building"], "building_name"),
(["room"], "room"),
(["trailer"], "trailer_number"),
(["walking", "directions"], "walking_directions"),
(["map"], "map_link"),
(["parking"], "parking_location"),
(["photo", "image", "picture"], "photo_path"),
(["route"], "_route"),
(["sales", "rep"], "_sales_rep"),
(["barcode", "tag", "qr"], "_barcode"),
]
def _normalize(s: str) -> str:
"""Lowercase, strip, collapse whitespace."""
return " ".join(str(s).lower().strip().split())
"""Lowercase, strip, collapse whitespace. Replace underscores with spaces
so 'machine_id' matches 'machine id'."""
return " ".join(str(s).lower().strip().replace("_", " ").replace("#", " ").split())
def map_columns(excel_headers: list[str]) -> dict[str, str]:
@@ -146,10 +168,17 @@ def map_columns(excel_headers: list[str]) -> dict[str, str]:
if not norm:
continue
matched = False
norm_words = set(norm.split())
for keywords, canonical in COLUMN_HEURISTICS:
for kw in keywords:
# Match if the header contains the keyword or vice versa
if kw in norm or norm in kw:
kw_norm = _normalize(kw)
if not kw_norm:
continue
kw_words = set(kw_norm.split())
# Match: all keyword words must be present in the header.
# This prevents short headers like "name" from matching
# multi-word keywords like "location name".
if kw_words.issubset(norm_words):
mapping[header] = canonical
discovered.append(f"{header}{canonical}")
matched = True