Add diff tracking — compare exports between runs
- New cantaloupe/diff.py: diff engine comparing xlsx by Asset ID, manifest tracking at ~/.cantaloupe-exports/manifest.json - New subcommand: python -m cantaloupe diff [--from FILE] [--to FILE] - Export --diff flag: auto-compare with previous export after download - Every export auto-records to manifest for history tracking - 44 new tests, 130/130 pass total
This commit is contained in:
@@ -3,11 +3,15 @@ Click CLI for cantaloupe export tool.
|
||||
|
||||
Commands:
|
||||
export Download machine list Excel from mycantaloupe.com
|
||||
diff Compare two machine list exports and show what changed
|
||||
configure Save credentials to ~/.cantaloupe.env
|
||||
|
||||
Usage:
|
||||
python -m cantaloupe export --output ./exports/
|
||||
python -m cantaloupe export --scheduled --output /data/exports/
|
||||
python -m cantaloupe export --diff # auto-compare with previous export
|
||||
python -m cantaloupe diff --from old.xlsx --to new.xlsx
|
||||
python -m cantaloupe diff # compare two most recent exports
|
||||
python -m cantaloupe configure
|
||||
"""
|
||||
|
||||
@@ -16,6 +20,7 @@ import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
|
||||
@@ -83,6 +88,54 @@ def _resolve_output(output: str | None, scheduled: bool) -> Path:
|
||||
return out_path
|
||||
|
||||
|
||||
def _print_diff_result(result: dict, from_label: str, to_label: str) -> None:
|
||||
"""Pretty-print a diff result to the console."""
|
||||
click.echo()
|
||||
click.echo("═" * 60)
|
||||
click.echo(" Diff Report")
|
||||
click.echo("═" * 60)
|
||||
click.echo(f" From : {from_label} ({result['from_count']} rows)")
|
||||
click.echo(f" To : {to_label} ({result['to_count']} rows)")
|
||||
click.echo(f" Key : {result['id_column']}")
|
||||
click.echo()
|
||||
|
||||
added = result["added_count"]
|
||||
removed = result["removed_count"]
|
||||
changed = result["changed_count"]
|
||||
|
||||
if added == 0 and removed == 0 and changed == 0:
|
||||
click.echo(" ✅ No changes — exports are identical.")
|
||||
else:
|
||||
if added:
|
||||
click.echo(f" 🟢 {added} machine(s) added")
|
||||
for row in result["added"][:10]:
|
||||
rid = row.get(result["id_column"], "?")
|
||||
click.echo(f" + {rid}")
|
||||
if added > 10:
|
||||
click.echo(f" ... and {added - 10} more")
|
||||
|
||||
if removed:
|
||||
click.echo(f" 🔴 {removed} machine(s) removed")
|
||||
for row in result["removed"][:10]:
|
||||
rid = row.get(result["id_column"], "?")
|
||||
click.echo(f" - {rid}")
|
||||
if removed > 10:
|
||||
click.echo(f" ... and {removed - 10} more")
|
||||
|
||||
if changed:
|
||||
click.echo(f" 🟡 {changed} machine(s) changed")
|
||||
for entry in result["changed"][:5]:
|
||||
rid = entry["id"]
|
||||
field_names = list(entry["changes"].keys())
|
||||
click.echo(f" ~ {rid} ({', '.join(field_names[:5])})")
|
||||
if len(field_names) > 5:
|
||||
click.echo(f" +{len(field_names) - 5} more fields")
|
||||
if changed > 5:
|
||||
click.echo(f" ... and {changed - 5} more")
|
||||
|
||||
click.echo("═" * 60)
|
||||
|
||||
|
||||
# ── CLI group ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -137,6 +190,12 @@ def cli(ctx: click.Context, verbose: bool) -> None:
|
||||
is_flag=True,
|
||||
help="Non-interactive mode for cron usage — skips credential prompts, uses defaults.",
|
||||
)
|
||||
@click.option(
|
||||
"--diff/--no-diff",
|
||||
"diff_flag",
|
||||
default=False,
|
||||
help="Compare with the previous export and show changes after download.",
|
||||
)
|
||||
@click.option(
|
||||
"--email",
|
||||
default=None,
|
||||
@@ -155,6 +214,7 @@ def export(
|
||||
output: str | None,
|
||||
headless: bool,
|
||||
scheduled: bool,
|
||||
diff_flag: bool,
|
||||
email: str | None,
|
||||
password: str | None,
|
||||
) -> None:
|
||||
@@ -163,6 +223,8 @@ def export(
|
||||
Authenticates with your Cantaloupe account and downloads the
|
||||
VueMachineList ExcelExport file, reporting file size, row count,
|
||||
and elapsed time on completion.
|
||||
|
||||
Use --diff to auto-compare with the previous export after download.
|
||||
"""
|
||||
# Resolve credentials (CLI flags > env vars > config file)
|
||||
resolved_email = email or ctx.obj.get("email", "")
|
||||
@@ -238,6 +300,32 @@ def export(
|
||||
click.echo(f" Rows : {row_count:,}")
|
||||
click.echo(f" Time : {elapsed:.1f}s")
|
||||
|
||||
# ── record to manifest ─────────────────────────────────────────────
|
||||
from cantaloupe.diff import record_export
|
||||
|
||||
# Snapshot the previous export *before* recording this one, so
|
||||
# we diff against the run-before-last, not this run.
|
||||
previous_path: Path | None = None
|
||||
if diff_flag:
|
||||
from cantaloupe.diff import get_previous_export
|
||||
previous_path = get_previous_export()
|
||||
|
||||
record_export(out_path, row_count, file_size)
|
||||
|
||||
# ── diff against previous export ────────────────────────────────────
|
||||
if diff_flag:
|
||||
if previous_path is None:
|
||||
click.echo("\n⚠ No previous export found — skipping diff.")
|
||||
elif previous_path.resolve() == out_path.resolve():
|
||||
click.echo("\n⚠ Previous export is the same file — skipping diff.")
|
||||
else:
|
||||
try:
|
||||
from cantaloupe.diff import diff_excels
|
||||
result = diff_excels(previous_path, out_path)
|
||||
_print_diff_result(result, previous_path.name, out_path.name)
|
||||
except Exception as e:
|
||||
click.echo(f"\n⚠ Diff failed: {e}", err=True)
|
||||
|
||||
|
||||
# ── configure command ───────────────────────────────────────────────────────
|
||||
|
||||
@@ -268,6 +356,86 @@ def configure(email: str, password: str) -> None:
|
||||
click.echo(f" You can now run: python -m cantaloupe export")
|
||||
|
||||
|
||||
# ── diff command ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
"--from", "from_path",
|
||||
default=None,
|
||||
help="Path to the older xlsx file (default: second-most-recent export).",
|
||||
)
|
||||
@click.option(
|
||||
"--to", "to_path",
|
||||
default=None,
|
||||
help="Path to the newer xlsx file (default: most recent export).",
|
||||
)
|
||||
@click.option(
|
||||
"--id-column",
|
||||
default=None,
|
||||
help="Column name to use as unique machine identifier (auto-detected if omitted).",
|
||||
)
|
||||
@click.option(
|
||||
"--json", "json_output",
|
||||
is_flag=True,
|
||||
help="Output the diff result as JSON instead of pretty-printed text.",
|
||||
)
|
||||
def diff(
|
||||
from_path: str | None,
|
||||
to_path: str | None,
|
||||
id_column: str | None,
|
||||
json_output: bool,
|
||||
) -> None:
|
||||
"""Compare two machine list Excel exports and show what changed.
|
||||
|
||||
If --from and --to are omitted, the two most recent exports are used
|
||||
(tracked via ~/.cantaloupe-exports/manifest.json).
|
||||
|
||||
Reports machines added, removed, and changed — keyed on the Asset ID
|
||||
column (auto-detected, or specified with --id-column).
|
||||
"""
|
||||
from cantaloupe.diff import diff_excels, get_latest_exports
|
||||
|
||||
# Resolve file paths
|
||||
if from_path and to_path:
|
||||
f_path = Path(os.path.expanduser(from_path)).resolve()
|
||||
t_path = Path(os.path.expanduser(to_path)).resolve()
|
||||
elif from_path or to_path:
|
||||
raise click.UsageError(
|
||||
"Specify both --from and --to, or neither (to use latest exports)."
|
||||
)
|
||||
else:
|
||||
latest = get_latest_exports(2)
|
||||
if len(latest) < 2:
|
||||
click.echo(
|
||||
"ERROR: Need at least two exports to diff. "
|
||||
"Run an export first, or specify --from and --to explicitly.",
|
||||
err=True,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
f_path = latest[1] # older
|
||||
t_path = latest[0] # newer
|
||||
|
||||
if not f_path.exists():
|
||||
click.echo(f"ERROR: File not found: {f_path}", err=True)
|
||||
raise SystemExit(1)
|
||||
if not t_path.exists():
|
||||
click.echo(f"ERROR: File not found: {t_path}", err=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
try:
|
||||
result = diff_excels(f_path, t_path, id_column=id_column)
|
||||
except (ValueError, FileNotFoundError) as e:
|
||||
click.echo(f"ERROR: {e}", err=True)
|
||||
raise SystemExit(1)
|
||||
|
||||
if json_output:
|
||||
import json
|
||||
click.echo(json.dumps(result, indent=2, default=str))
|
||||
else:
|
||||
_print_diff_result(result, f_path.name, t_path.name)
|
||||
|
||||
|
||||
# ── direct invocation ───────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""
|
||||
Diff engine for comparing Cantaloupe machine list Excel exports.
|
||||
|
||||
Reports:
|
||||
- 🟢 New machines added (present in new, not in old)
|
||||
- 🔴 Machines removed (present in old, not in new)
|
||||
- 🟡 Machines with changed fields (same ID, different values)
|
||||
|
||||
Also manages a JSON manifest at ~/.cantaloupe-exports/manifest.json
|
||||
that tracks export history for auto-diff functionality.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import openpyxl
|
||||
|
||||
MANIFEST_DIR = Path.home() / ".cantaloupe-exports"
|
||||
MANIFEST_FILE = MANIFEST_DIR / "manifest.json"
|
||||
|
||||
|
||||
# ── xlsx reading ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _read_xlsx(path: Path) -> tuple[list[str], list[dict]]:
|
||||
"""Read an xlsx file and return (headers, list of row dicts).
|
||||
|
||||
Each row dict maps header name → cell value.
|
||||
Returns empty lists if the file is empty or unreadable.
|
||||
"""
|
||||
wb = openpyxl.load_workbook(path, read_only=True)
|
||||
try:
|
||||
ws = wb.active
|
||||
if ws is None:
|
||||
return [], []
|
||||
rows = list(ws.iter_rows(values_only=True))
|
||||
finally:
|
||||
wb.close()
|
||||
|
||||
if not rows:
|
||||
return [], []
|
||||
|
||||
headers = [str(h) if h is not None else "" for h in rows[0]]
|
||||
data = []
|
||||
for row in rows[1:]:
|
||||
row_dict = {}
|
||||
for i, cell in enumerate(row):
|
||||
if i < len(headers):
|
||||
row_dict[headers[i]] = cell
|
||||
data.append(row_dict)
|
||||
|
||||
return headers, data
|
||||
|
||||
|
||||
def _find_id_column(headers: list[str]) -> str:
|
||||
"""Find the best unique-identifier column from headers.
|
||||
|
||||
Priority:
|
||||
1. Column whose name contains 'asset' AND 'id' (case-insensitive)
|
||||
2. Column whose name equals 'id' (case-insensitive)
|
||||
3. First column (fallback)
|
||||
"""
|
||||
for h in headers:
|
||||
hl = h.lower()
|
||||
if "asset" in hl and "id" in hl:
|
||||
return h
|
||||
for h in headers:
|
||||
if h.lower() == "id":
|
||||
return h
|
||||
return headers[0] if headers else ""
|
||||
|
||||
|
||||
# ── diff engine ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _normalize(val) -> str:
|
||||
"""Convert a cell value to a string for comparison and keying."""
|
||||
if val is None:
|
||||
return ""
|
||||
return str(val).strip()
|
||||
|
||||
|
||||
def diff_excels(
|
||||
from_path: Path,
|
||||
to_path: Path,
|
||||
id_column: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Compare two xlsx files and return a structured diff result.
|
||||
|
||||
Args:
|
||||
from_path: Path to the older ("from") xlsx file.
|
||||
to_path: Path to the newer ("to") xlsx file.
|
||||
id_column: Column name to use as the unique machine identifier.
|
||||
Auto-detected if not provided (looks for 'Asset ID').
|
||||
|
||||
Returns:
|
||||
dict with keys:
|
||||
- id_column: str — column used as the key
|
||||
- from_path, to_path: str — file paths
|
||||
- from_count, to_count: int — data row counts
|
||||
- added: list[dict] — rows present in 'to' but not 'from'
|
||||
- removed: list[dict] — rows present in 'from' but not 'to'
|
||||
- changed: list[dict] — {id, changes: {col: {old, new}}}
|
||||
- added_count, removed_count, changed_count: int
|
||||
|
||||
Raises:
|
||||
ValueError: if either file has no data / no rows.
|
||||
FileNotFoundError: if either file doesn't exist.
|
||||
"""
|
||||
from_headers, from_data = _read_xlsx(from_path)
|
||||
to_headers, to_data = _read_xlsx(to_path)
|
||||
|
||||
if not to_headers:
|
||||
raise ValueError(f"No data found in {to_path}")
|
||||
|
||||
# Determine id column (prefer the 'to' file's headers)
|
||||
id_col = id_column or _find_id_column(to_headers)
|
||||
|
||||
# Build lookup dicts keyed on the id column
|
||||
from_by_id: dict[str, dict] = {}
|
||||
for row in from_data:
|
||||
key = _normalize(row.get(id_col, ""))
|
||||
from_by_id[key] = row
|
||||
|
||||
to_by_id: dict[str, dict] = {}
|
||||
for row in to_data:
|
||||
key = _normalize(row.get(id_col, ""))
|
||||
to_by_id[key] = row
|
||||
|
||||
from_ids = set(from_by_id.keys())
|
||||
to_ids = set(to_by_id.keys())
|
||||
|
||||
added_ids = to_ids - from_ids
|
||||
removed_ids = from_ids - to_ids
|
||||
common_ids = from_ids & to_ids
|
||||
|
||||
# Collect changed rows
|
||||
changed: list[dict] = []
|
||||
# Use headers from 'to' file as canonical
|
||||
for cid in sorted(common_ids):
|
||||
old_row = from_by_id[cid]
|
||||
new_row = to_by_id[cid]
|
||||
field_changes: dict[str, dict[str, object]] = {}
|
||||
for col in to_headers:
|
||||
old_val = _normalize(old_row.get(col))
|
||||
new_val = _normalize(new_row.get(col))
|
||||
if old_val != new_val:
|
||||
field_changes[col] = {"old": old_row.get(col), "new": new_row.get(col)}
|
||||
if field_changes:
|
||||
changed.append({"id": cid, "changes": field_changes})
|
||||
|
||||
return {
|
||||
"id_column": id_col,
|
||||
"from_path": str(from_path),
|
||||
"to_path": str(to_path),
|
||||
"from_count": len(from_data),
|
||||
"to_count": len(to_data),
|
||||
"added": [to_by_id[aid] for aid in sorted(added_ids)],
|
||||
"removed": [from_by_id[rid] for rid in sorted(removed_ids)],
|
||||
"changed": changed,
|
||||
"added_count": len(added_ids),
|
||||
"removed_count": len(removed_ids),
|
||||
"changed_count": len(changed),
|
||||
}
|
||||
|
||||
|
||||
# ── manifest management ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_manifest() -> dict:
|
||||
"""Load the export history manifest from ~/.cantaloupe-exports/manifest.json.
|
||||
|
||||
Returns a dict with an 'exports' list. Creates an empty manifest
|
||||
if the file doesn't exist.
|
||||
"""
|
||||
MANIFEST_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if MANIFEST_FILE.exists():
|
||||
try:
|
||||
return json.loads(MANIFEST_FILE.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {"exports": []}
|
||||
return {"exports": []}
|
||||
|
||||
|
||||
def save_manifest(manifest: dict) -> None:
|
||||
"""Save the manifest to disk."""
|
||||
MANIFEST_DIR.mkdir(parents=True, exist_ok=True)
|
||||
MANIFEST_FILE.write_text(json.dumps(manifest, indent=2, default=str) + "\n")
|
||||
|
||||
|
||||
def record_export(
|
||||
filepath: Path,
|
||||
row_count: int,
|
||||
file_size: int,
|
||||
) -> None:
|
||||
"""Record a completed export in the manifest."""
|
||||
manifest = load_manifest()
|
||||
manifest.setdefault("exports", []).append(
|
||||
{
|
||||
"path": str(filepath),
|
||||
"filename": filepath.name,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"row_count": row_count,
|
||||
"file_size": file_size,
|
||||
}
|
||||
)
|
||||
save_manifest(manifest)
|
||||
|
||||
|
||||
def get_previous_export() -> Optional[Path]:
|
||||
"""Return the path of the most recent export, or None if no history."""
|
||||
manifest = load_manifest()
|
||||
exports = manifest.get("exports", [])
|
||||
if not exports:
|
||||
return None
|
||||
previous = exports[-1]["path"]
|
||||
p = Path(previous)
|
||||
if p.exists():
|
||||
return p
|
||||
# File was moved/deleted — return None so we don't diff against a ghost
|
||||
return None
|
||||
|
||||
|
||||
def get_latest_exports(n: int = 2) -> list[Path]:
|
||||
"""Return paths of the N most recent existing exports, newest first."""
|
||||
manifest = load_manifest()
|
||||
exports = manifest.get("exports", [])
|
||||
result: list[Path] = []
|
||||
for entry in reversed(exports):
|
||||
p = Path(entry["path"])
|
||||
if p.exists():
|
||||
result.append(p)
|
||||
if len(result) >= n:
|
||||
break
|
||||
return result
|
||||
Reference in New Issue
Block a user