cantaloupe_sync: fix column mapping heuristic, add comprehensive tests
- Fix greedy matching where 'name' keyword matched 'Location Name' header - Use word-subset matching instead of substring matching - Normalize keywords (replace _ and # with spaces) to match header normalization - Reorder heuristics: multi-word specific patterns before single-word generics - Add 10 unit tests for mapping, diff, and Excel parsing - Add 23 integration tests for all sync API endpoints - All 33 tests pass
This commit is contained in:
+54
-25
@@ -98,37 +98,59 @@ ASSET_COLUMNS = [
|
||||
COLUMN_HEURISTICS = [
|
||||
# machine_id is the primary key for upserts
|
||||
(["asset id", "assetid", "machine id", "machineid", "machine_id", "equipment id", "unit id", "asset number"], "machine_id"),
|
||||
(["serial", "serial number", "serialnumber", "serial_no", "s/n"], "serial_number"),
|
||||
(["name", "asset name", "equipment name", "description"], "name"),
|
||||
(["description", "notes", "detail", "comments"], "description"),
|
||||
# Multi-word / specific heuristics first to avoid greedy single-word matches
|
||||
(["location name", "location_name", "site name"], "_location_name"),
|
||||
(["building name", "building_name", "bldg name"], "building_name"),
|
||||
(["building number", "building_number", "building_no", "bldg number", "bldg #", "building #"], "building_number"),
|
||||
(["trailer number", "trailer_number", "trailer_no", "trailer #", "mobile unit"], "trailer_number"),
|
||||
(["walking directions", "walking_directions", "how to get there"], "walking_directions"),
|
||||
(["map link", "map_link", "google maps", "map url", "location url"], "map_link"),
|
||||
(["parking location", "parking_location", "parking spot"], "parking_location"),
|
||||
(["street address", "location address", "site address"], "address"),
|
||||
(["asset name", "equipment name"], "name"),
|
||||
(["model number", "model_no", "model #"], "model"),
|
||||
(["serial number", "serialnumber", "serial_no", "s/n"], "serial_number"),
|
||||
(["room number", "room #", "suite"], "room"),
|
||||
(["customer name", "client name", "account name", "company"], "_customer_name"),
|
||||
(["photo path", "photo_path", "image path"], "photo_path"),
|
||||
(["route number", "route #", "delivery route"], "_route"),
|
||||
(["sales rep", "salesperson", "account manager"], "_sales_rep"),
|
||||
(["qr code", "tag number", "tag #"], "_barcode"),
|
||||
(["geofence radius", "geofence_radius", "geo radius"], "geofence_radius_meters"),
|
||||
(["gps lat", "lat"], "latitude"),
|
||||
(["gps lng", "gps long", "long", "lng", "lon"], "longitude"),
|
||||
# Single-word/generic heuristics (last, after specific multi-word ones)
|
||||
(["serial", "s/n"], "serial_number"),
|
||||
(["category", "type", "asset type", "equipment type", "class"], "category"),
|
||||
(["status", "state", "condition"], "status"),
|
||||
(["make", "manufacturer", "brand", "vendor"], "make"),
|
||||
(["model", "model number", "model_no", "model #"], "model"),
|
||||
(["address", "street", "street address", "location address", "site address"], "address"),
|
||||
(["building name", "building_name", "bldg name", "site name"], "building_name"),
|
||||
(["building number", "building_number", "building_no", "bldg number", "bldg #", "building #"], "building_number"),
|
||||
(["address", "street"], "address"),
|
||||
(["floor", "level", "story"], "floor"),
|
||||
(["room", "room number", "room #", "suite"], "room"),
|
||||
(["trailer", "trailer number", "trailer_no", "trailer #", "mobile unit"], "trailer_number"),
|
||||
(["walking directions", "walking_directions", "directions", "how to get there"], "walking_directions"),
|
||||
(["map link", "map_link", "google maps", "map url", "location url"], "map_link"),
|
||||
(["parking", "parking location", "parking_location", "parking spot"], "parking_location"),
|
||||
(["latitude", "lat", "gps lat"], "latitude"),
|
||||
(["longitude", "long", "lng", "lon", "gps lng", "gps long"], "longitude"),
|
||||
(["geofence", "geo radius", "radius", "geofence radius", "geofence_radius"], "geofence_radius_meters"),
|
||||
(["customer", "client", "account", "company", "customer name"], "_customer_name"),
|
||||
(["location name", "location_name", "site", "site name", "location"], "_location_name"),
|
||||
(["photo", "photo path", "photo_path", "image", "picture"], "photo_path"),
|
||||
(["route", "route number", "route #", "delivery route"], "_route"),
|
||||
(["sales", "sales rep", "salesperson", "account manager"], "_sales_rep"),
|
||||
(["barcode", "qr code", "tag", "tag number", "tag #"], "_barcode"),
|
||||
(["latitude", "gps"], "latitude"),
|
||||
(["longitude", "geo"], "longitude"),
|
||||
(["geofence", "radius"], "geofence_radius_meters"),
|
||||
(["customer", "client", "account"], "_customer_name"),
|
||||
(["location", "site"], "_location_name"),
|
||||
(["name"], "name"),
|
||||
(["description", "notes", "detail", "comments"], "description"),
|
||||
(["model"], "model"),
|
||||
(["building"], "building_name"),
|
||||
(["room"], "room"),
|
||||
(["trailer"], "trailer_number"),
|
||||
(["walking", "directions"], "walking_directions"),
|
||||
(["map"], "map_link"),
|
||||
(["parking"], "parking_location"),
|
||||
(["photo", "image", "picture"], "photo_path"),
|
||||
(["route"], "_route"),
|
||||
(["sales", "rep"], "_sales_rep"),
|
||||
(["barcode", "tag", "qr"], "_barcode"),
|
||||
]
|
||||
|
||||
|
||||
def _normalize(s: str) -> str:
|
||||
"""Lowercase, strip, collapse whitespace."""
|
||||
return " ".join(str(s).lower().strip().split())
|
||||
"""Lowercase, strip, collapse whitespace. Replace underscores with spaces
|
||||
so 'machine_id' matches 'machine id'."""
|
||||
return " ".join(str(s).lower().strip().replace("_", " ").replace("#", " ").split())
|
||||
|
||||
|
||||
def map_columns(excel_headers: list[str]) -> dict[str, str]:
|
||||
@@ -146,10 +168,17 @@ def map_columns(excel_headers: list[str]) -> dict[str, str]:
|
||||
if not norm:
|
||||
continue
|
||||
matched = False
|
||||
norm_words = set(norm.split())
|
||||
for keywords, canonical in COLUMN_HEURISTICS:
|
||||
for kw in keywords:
|
||||
# Match if the header contains the keyword or vice versa
|
||||
if kw in norm or norm in kw:
|
||||
kw_norm = _normalize(kw)
|
||||
if not kw_norm:
|
||||
continue
|
||||
kw_words = set(kw_norm.split())
|
||||
# Match: all keyword words must be present in the header.
|
||||
# This prevents short headers like "name" from matching
|
||||
# multi-word keywords like "location name".
|
||||
if kw_words.issubset(norm_words):
|
||||
mapping[header] = canonical
|
||||
discovered.append(f"{header} → {canonical}")
|
||||
matched = True
|
||||
|
||||
Reference in New Issue
Block a user