🌱 Complete seed import tool — all 17 kanban tasks done
Modules: - parser.py: Full Excel parser (1,848 machines, Disney mapping, Pattern A+B parsing) - db_writer.py: SQLite writer with per-field update policy (seed + update modes) - backup.py: GPS/photo backup, restore, and compare - reporter.py: Comprehensive validation report generator - main.py: CLI entry point Database: assets.db with 1,848 machines across 36 columns Handbook: reference_handbook.md v2.0 — all sections finalized Report: validation_report.md — 184 OCR corrections, 143 unknown machines
This commit is contained in:
+480
@@ -0,0 +1,480 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🌱 Canteen Seed Data Import — Database Writer
|
||||
|
||||
Writes parsed machine records to SQLite database with per-field update policy.
|
||||
|
||||
Usage:
|
||||
from db_writer import DatabaseWriter
|
||||
writer = DatabaseWriter('seed-data/assets.db')
|
||||
writer.seed(machines) # First import — insert all fresh
|
||||
writer.update(machines) # Subsequent import — upsert with preserve/auto rules
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
# ─── Per-field update policy (from Handbook Section 11) ───────────────────────
|
||||
|
||||
# Fields that are NEVER overwritten if they already have a value
|
||||
PRESERVE_FIELDS = {
|
||||
"latitude",
|
||||
"longitude",
|
||||
"photo_path",
|
||||
"install_date",
|
||||
"deployed",
|
||||
"pulled_date",
|
||||
}
|
||||
|
||||
# Fields that are ALWAYS overwritten from new data
|
||||
AUTO_FIELDS = {
|
||||
"serial_number",
|
||||
"name",
|
||||
"make",
|
||||
"model",
|
||||
"class",
|
||||
"is_glass_front",
|
||||
"company",
|
||||
"place",
|
||||
"building_name",
|
||||
"floor",
|
||||
"zone",
|
||||
"zone_type",
|
||||
"address",
|
||||
"location_area",
|
||||
"dex_report_date",
|
||||
"disney_park",
|
||||
"remote_pricing_status",
|
||||
"alerts",
|
||||
"telemetry_provider",
|
||||
"card_reader_brand",
|
||||
"priority",
|
||||
"yearly_sales",
|
||||
"monthly_sales",
|
||||
"weekly_sales",
|
||||
"days_since_restock",
|
||||
"prepick_group",
|
||||
"has_cashless",
|
||||
}
|
||||
|
||||
# All 33 DB columns (Section 11 of the handbook)
|
||||
# machine_id is the primary key — handled separately
|
||||
ALL_COLUMNS = [
|
||||
"machine_id", # preserve — primary key
|
||||
"serial_number", # auto
|
||||
"name", # auto (generated: "Make @ Place")
|
||||
"make", # auto
|
||||
"model", # auto
|
||||
"class", # auto
|
||||
"is_glass_front", # auto
|
||||
"company", # auto
|
||||
"place", # auto
|
||||
"building_name", # auto
|
||||
"floor", # auto
|
||||
"zone", # auto
|
||||
"zone_type", # auto
|
||||
"address", # auto
|
||||
"location_area", # auto
|
||||
"latitude", # preserve
|
||||
"longitude", # preserve
|
||||
"photo_path", # preserve
|
||||
"dex_report_date", # auto
|
||||
"install_date", # preserve
|
||||
"deployed", # preserve
|
||||
"pulled_date", # preserve
|
||||
"disney_park", # auto
|
||||
"remote_pricing_status", # auto
|
||||
"alerts", # auto
|
||||
"telemetry_provider", # auto
|
||||
"card_reader_brand", # auto
|
||||
"priority", # auto
|
||||
"yearly_sales", # auto
|
||||
"monthly_sales", # auto
|
||||
"weekly_sales", # auto
|
||||
"days_since_restock", # auto
|
||||
"prepick_group", # auto
|
||||
"has_cashless", # auto
|
||||
]
|
||||
|
||||
# SQL column types
|
||||
COLUMN_TYPES = {
|
||||
"machine_id": "TEXT PRIMARY KEY",
|
||||
"serial_number": "TEXT",
|
||||
"name": "TEXT",
|
||||
"make": "TEXT",
|
||||
"model": "TEXT",
|
||||
"class": "TEXT",
|
||||
"is_glass_front": "INTEGER",
|
||||
"company": "TEXT",
|
||||
"place": "TEXT",
|
||||
"building_name": "TEXT",
|
||||
"floor": "INTEGER",
|
||||
"zone": "TEXT",
|
||||
"zone_type": "TEXT",
|
||||
"address": "TEXT",
|
||||
"location_area": "TEXT",
|
||||
"latitude": "REAL",
|
||||
"longitude": "REAL",
|
||||
"photo_path": "TEXT",
|
||||
"dex_report_date": "TEXT",
|
||||
"install_date": "TEXT",
|
||||
"deployed": "INTEGER",
|
||||
"pulled_date": "TEXT",
|
||||
"disney_park": "TEXT",
|
||||
"remote_pricing_status": "TEXT",
|
||||
"alerts": "TEXT",
|
||||
"telemetry_provider": "TEXT",
|
||||
"card_reader_brand": "TEXT",
|
||||
"priority": "TEXT",
|
||||
"yearly_sales": "REAL",
|
||||
"monthly_sales": "REAL",
|
||||
"weekly_sales": "REAL",
|
||||
"days_since_restock": "INTEGER",
|
||||
"prepick_group": "TEXT",
|
||||
"has_cashless": "INTEGER",
|
||||
# Extra metadata fields (not in Section 11 but useful)
|
||||
"created_at": "TEXT",
|
||||
"updated_at": "TEXT",
|
||||
}
|
||||
|
||||
# Extra columns for metadata
|
||||
EXTRA_COLUMNS = ["created_at", "updated_at"]
|
||||
|
||||
# Fields that are not in the parser output and only come from external sources
|
||||
EXTERNAL_PRESERVE_FIELDS = {"latitude", "longitude", "photo_path"}
|
||||
|
||||
|
||||
def _serialize_value(val):
|
||||
"""Convert a Python value to a SQLite-friendly value."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dict):
|
||||
return json.dumps(val, default=str)
|
||||
if isinstance(val, bool):
|
||||
return 1 if val else 0
|
||||
if isinstance(val, datetime):
|
||||
return val.isoformat()
|
||||
return val
|
||||
|
||||
|
||||
def _deserialize_value(val, column):
|
||||
"""Convert SQLite value back to Python, if needed."""
|
||||
if column == "is_glass_front" or column == "has_cashless" or column == "deployed":
|
||||
if val is None:
|
||||
return None
|
||||
return bool(val)
|
||||
if column == "alerts" and val is not None and isinstance(val, str):
|
||||
try:
|
||||
return json.loads(val)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return val
|
||||
return val
|
||||
|
||||
|
||||
def generate_name(make, place):
|
||||
"""Generate a machine name from Make and Place."""
|
||||
make_str = str(make).strip() if make else "Unknown"
|
||||
place_str = str(place).strip() if place else "Unknown"
|
||||
return f"{make_str} @ {place_str}"
|
||||
|
||||
|
||||
class DatabaseWriter:
|
||||
"""
|
||||
SQLite database writer for seed data import.
|
||||
|
||||
Handles both 'seed' (fresh insert) and 'update' (upsert with policy) modes.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path):
|
||||
self.db_path = os.path.abspath(db_path)
|
||||
self.conn = None
|
||||
self._open_connection()
|
||||
|
||||
def _open_connection(self):
|
||||
"""Open SQLite connection, creating dir if needed."""
|
||||
db_dir = os.path.dirname(self.db_path)
|
||||
if db_dir and not os.path.exists(db_dir):
|
||||
os.makedirs(db_dir, exist_ok=True)
|
||||
self.conn = sqlite3.connect(self.db_path)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
|
||||
def create_table(self):
|
||||
"""Create the 'assets' table with all columns from Section 11."""
|
||||
columns_defs = []
|
||||
for col in ALL_COLUMNS:
|
||||
col_type = COLUMN_TYPES.get(col, "TEXT")
|
||||
columns_defs.append(f" {col} {col_type}")
|
||||
for col in EXTRA_COLUMNS:
|
||||
col_type = COLUMN_TYPES.get(col, "TEXT")
|
||||
columns_defs.append(f" {col} {col_type}")
|
||||
|
||||
create_sql = (
|
||||
"CREATE TABLE IF NOT EXISTS assets (\n"
|
||||
+ ",\n".join(columns_defs)
|
||||
+ "\n)"
|
||||
)
|
||||
self.conn.execute(create_sql)
|
||||
self.conn.commit()
|
||||
|
||||
def _extract_row_values(self, machine):
|
||||
"""Extract values from a machine dict, mapping to DB columns."""
|
||||
now = datetime.utcnow().isoformat()
|
||||
|
||||
# Generate name from Make + Place
|
||||
name = generate_name(machine.get("make"), machine.get("place"))
|
||||
|
||||
# Build row dict (only the columns we store)
|
||||
row = {}
|
||||
|
||||
for col in ALL_COLUMNS:
|
||||
if col == "name":
|
||||
row[col] = name
|
||||
elif col == "machine_id":
|
||||
row[col] = str(machine.get("machine_id", ""))
|
||||
elif col == "is_glass_front":
|
||||
row[col] = 1 if machine.get("is_glass_front") else 0
|
||||
elif col == "has_cashless":
|
||||
val = machine.get("has_cashless")
|
||||
if val is True or val == 1 or val == "Yes" or val == "yes":
|
||||
row[col] = 1
|
||||
elif val is False or val == 0 or val == "No" or val == "no":
|
||||
row[col] = 0
|
||||
else:
|
||||
row[col] = None
|
||||
elif col == "deployed":
|
||||
val = machine.get("deployed")
|
||||
if val is True or val == 1 or val == "Yes" or val == "yes":
|
||||
row[col] = 1
|
||||
elif val is False or val == 0 or val == "No" or val == "no":
|
||||
row[col] = 0
|
||||
else:
|
||||
row[col] = None
|
||||
elif col == "alerts":
|
||||
row[col] = _serialize_value(machine.get("alerts"))
|
||||
elif col == "floor":
|
||||
val = machine.get("floor")
|
||||
if val is not None:
|
||||
try:
|
||||
row[col] = int(val)
|
||||
except (ValueError, TypeError):
|
||||
row[col] = None
|
||||
else:
|
||||
row[col] = None
|
||||
elif col in ("yearly_sales", "monthly_sales", "weekly_sales"):
|
||||
row[col] = _serialize_value(machine.get(col))
|
||||
elif col in ("latitude", "longitude"):
|
||||
row[col] = _serialize_value(machine.get(col))
|
||||
else:
|
||||
val = machine.get(col)
|
||||
if val is not None and not isinstance(val, (str, int, float, bool)):
|
||||
val = str(val)
|
||||
row[col] = val
|
||||
|
||||
# Extra metadata
|
||||
row["created_at"] = now
|
||||
row["updated_at"] = now
|
||||
|
||||
return row
|
||||
|
||||
def _columns_for_insert(self):
|
||||
"""Return list of column names for INSERT (excluding PRIMARY KEY metadata for upsert)."""
|
||||
return ALL_COLUMNS + EXTRA_COLUMNS
|
||||
|
||||
def seed(self, machines):
|
||||
"""
|
||||
Seed mode: Insert all machines fresh.
|
||||
Drops any existing data and re-inserts.
|
||||
|
||||
Returns dict with counts.
|
||||
"""
|
||||
self.create_table()
|
||||
|
||||
# Clear existing data
|
||||
self.conn.execute("DELETE FROM assets")
|
||||
|
||||
insert_cols = self._columns_for_insert()
|
||||
placeholders = ", ".join("?" for _ in insert_cols)
|
||||
col_names = ", ".join(insert_cols)
|
||||
insert_sql = f"INSERT INTO assets ({col_names}) VALUES ({placeholders})"
|
||||
|
||||
count = 0
|
||||
errors = 0
|
||||
for machine in machines:
|
||||
try:
|
||||
row = self._extract_row_values(machine)
|
||||
values = [row.get(c) for c in insert_cols]
|
||||
self.conn.execute(insert_sql, values)
|
||||
count += 1
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
mid = machine.get("machine_id", "???")
|
||||
print(f" ⚠️ Error inserting machine {mid}: {e}")
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
return {
|
||||
"inserted": count,
|
||||
"errors": errors,
|
||||
"total": len(machines),
|
||||
}
|
||||
|
||||
def update(self, machines):
|
||||
"""
|
||||
Update mode: Upsert by machine_id.
|
||||
|
||||
PRESERVE fields (lat, lng, photo_path, install_date, deployed, pulled_date):
|
||||
Skip update if existing value is non-null.
|
||||
AUTO fields: Always overwrite.
|
||||
machine_id: Never changes (used as match key).
|
||||
|
||||
Returns dict with counts.
|
||||
"""
|
||||
self.create_table()
|
||||
|
||||
# Ensure machine_id is indexed
|
||||
self.conn.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_assets_machine_id ON assets(machine_id)
|
||||
""")
|
||||
|
||||
insert_cols = self._columns_for_insert()
|
||||
col_names = ", ".join(insert_cols)
|
||||
|
||||
stats = {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"preserved": 0,
|
||||
"skipped": 0,
|
||||
"errors": 0,
|
||||
}
|
||||
|
||||
# Prepare the UPDATE SET clause: col = ? for each non-PK column
|
||||
update_cols = [c for c in insert_cols if c != "machine_id"]
|
||||
set_clause = ", ".join(f"{c} = ?" for c in update_cols)
|
||||
|
||||
for machine in machines:
|
||||
mid = machine.get("machine_id")
|
||||
if not mid:
|
||||
stats["skipped"] += 1
|
||||
continue
|
||||
|
||||
mid_str = str(mid)
|
||||
|
||||
# Check if machine exists
|
||||
cursor = self.conn.execute(
|
||||
"SELECT * FROM assets WHERE machine_id = ?", (mid_str,)
|
||||
)
|
||||
existing = cursor.fetchone()
|
||||
|
||||
row = self._extract_row_values(machine)
|
||||
|
||||
if existing is None:
|
||||
# INSERT new record
|
||||
values = [row.get(c) for c in insert_cols]
|
||||
try:
|
||||
self.conn.execute(
|
||||
f"INSERT INTO assets ({col_names}) VALUES ({', '.join('?' for _ in insert_cols)})",
|
||||
values,
|
||||
)
|
||||
stats["inserted"] += 1
|
||||
except Exception as e:
|
||||
stats["errors"] += 1
|
||||
print(f" ⚠️ Error inserting machine {mid}: {e}")
|
||||
else:
|
||||
# UPDATE existing record with preserve/auto rules
|
||||
existing_dict = dict(existing)
|
||||
updates = {}
|
||||
preserved_count = 0
|
||||
|
||||
for col in update_cols:
|
||||
new_val = row.get(col)
|
||||
old_val = existing_dict.get(col)
|
||||
|
||||
if col in PRESERVE_FIELDS:
|
||||
# PRESERVE: skip if existing value is non-null
|
||||
if old_val is not None and old_val != "":
|
||||
preserved_count += 1
|
||||
continue # Don't update
|
||||
# If old is NULL, we can set it
|
||||
updates[col] = new_val
|
||||
elif col == "machine_id":
|
||||
continue # PK — never update
|
||||
else:
|
||||
# AUTO: always overwrite
|
||||
updates[col] = new_val
|
||||
|
||||
if updates:
|
||||
# Build dynamic UPDATE
|
||||
set_parts = []
|
||||
set_values = []
|
||||
for col, val in updates.items():
|
||||
set_parts.append(f"{col} = ?")
|
||||
set_values.append(val)
|
||||
# Always update updated_at
|
||||
set_parts.append("updated_at = ?")
|
||||
set_values.append(datetime.utcnow().isoformat())
|
||||
set_values.append(mid_str)
|
||||
|
||||
try:
|
||||
self.conn.execute(
|
||||
f"UPDATE assets SET {', '.join(set_parts)} WHERE machine_id = ?",
|
||||
set_values,
|
||||
)
|
||||
stats["updated"] += 1
|
||||
stats["preserved"] += preserved_count
|
||||
except Exception as e:
|
||||
stats["errors"] += 1
|
||||
print(f" ⚠️ Error updating machine {mid}: {e}")
|
||||
else:
|
||||
stats["skipped"] += 1
|
||||
|
||||
self.conn.commit()
|
||||
return stats
|
||||
|
||||
def get_machine_count(self):
|
||||
"""Return the number of machines in the database."""
|
||||
cursor = self.conn.execute("SELECT COUNT(*) FROM assets")
|
||||
return cursor.fetchone()[0]
|
||||
|
||||
def close(self):
|
||||
"""Close the database connection."""
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
self.conn = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
|
||||
|
||||
def write_seed(machines, db_path):
|
||||
"""Convenience: seed mode with reporting."""
|
||||
print(f" 📁 Database: {db_path}")
|
||||
with DatabaseWriter(db_path) as writer:
|
||||
stats = writer.seed(machines)
|
||||
final_count = writer.get_machine_count()
|
||||
print(f" ✅ Seeded: {stats['inserted']} machines")
|
||||
print(f" 📊 Total in DB: {final_count}")
|
||||
if stats["errors"]:
|
||||
print(f" ⚠️ Errors: {stats['errors']}")
|
||||
return stats
|
||||
|
||||
|
||||
def write_update(machines, db_path):
|
||||
"""Convenience: update mode with reporting."""
|
||||
print(f" 📁 Database: {db_path}")
|
||||
with DatabaseWriter(db_path) as writer:
|
||||
stats = writer.update(machines)
|
||||
final_count = writer.get_machine_count()
|
||||
print(f" ✅ Inserted: {stats['inserted']}")
|
||||
print(f" 🔄 Updated: {stats['updated']}")
|
||||
print(f" 🛡️ Preserved fields: {stats['preserved']}")
|
||||
print(f" ⏭️ Skipped (no changes): {stats['skipped']}")
|
||||
print(f" 📊 Total in DB: {final_count}")
|
||||
if stats["errors"]:
|
||||
print(f" ⚠️ Errors: {stats['errors']}")
|
||||
return stats
|
||||
Reference in New Issue
Block a user