#!/usr/bin/env python3 """ cdcrop_tune.py - tune cdcrop's automatic angle choice to your own picks. python cdcrop_tune.py --scans "C:\\path\\to\\scans" --decisions "C:\\path\\to\\output\\cdcrop_decisions.csv" Runs on your computer with your scans, so the result matches exactly what the cropper does there. For every cover you reviewed in the interface, it measures everything the automatic choice weighs, then tries a few thousand weightings and keeps the one that reproduces your picks best - preferring settings in the middle of a good region, so small differences on new covers don't flip results. The winner is saved to cdcrop_tuning.json next to cdcrop.py and used automatically from then on (delete that file to go back to the built-in weights). Run it again whenever you've reviewed a new batch. A pick counts as reproduced when the automatic angle lands within --tol px (default 2) of yours, measured as drift across the whole cover. New weights are only saved if they predict covers they weren't tuned on (leave-one-out) better than the current ones - otherwise they'd just be memorising. """ import argparse import csv import itertools import json import math import os import sys import time import warnings import numpy as np sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import cdcrop # noqa: E402 DETAIL_GRID = {"DEEP_BLEND": (0.0, 0.5, 1.0), "SAL_POWER": (1.0, 2.0), "SAL_FLOOR": (50, 60, 70)} WEIGHT_GRID = {"W_V": (0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 4.0, 6.0, 8.0), "W_LEFT": (0.0, 0.05, 0.1, 0.2, 0.4), "W_EDGE": (0.0, 0.01, 0.02, 0.05), "W_BLUR": (0.0, 0.25, 0.5)} H_RATIO = 0.5 # W_H = W_V * H_RATIO (horizontal lines count half, as calibrated so far) def read_picks(path): """Latest reviewed angle per cover (discs are skipped - they're tuned by hand).""" picks = {} with open(path, newline="") as fh: for row in csv.DictReader(fh): cands = json.loads(row.get("candidates") or "{}") if "as scanned" in cands: # a disc continue picks[row["cover"]] = {"chosen": float(row["chosen_deg"]), "how": row["how"], "logged": {k: row.get(f"{k}_deg") for k in ("left", "right", "top", "bottom")}} return picks def analyse(scans_dir, picks, inset, cal): """angle_components for every reviewed cover, with one loss per detail variant.""" by_scan = {} for cid in picks: base, idx = cid.rsplit("_", 1) by_scan.setdefault(base, []).append(int(idx)) variants = [dict(zip(DETAIL_GRID, v)) for v in itertools.product(*DETAIL_GRID.values())] covers = {} for base, idxs in sorted(by_scan.items()): path = next((os.path.join(scans_dir, f) for f in os.listdir(scans_dir) if os.path.splitext(f)[0] == base and f.lower().endswith(".bmp")), None) if not path: print(f" !! {base}.bmp not found in the scans folder - skipping its covers") continue t0 = time.time() arr8, _, dpi = cdcrop.load_scan(path) arr, _ = cdcrop.true_space(arr8.astype(np.float32), cal) gray = arr[..., :3].mean(axis=2) if arr.ndim == 3 else arr objs = cdcrop.find_objects(arr, max(max(idxs), 2), dpi) for idx, (kind, rect, _) in enumerate(objs, start=1): cid = f"{base}_{idx}" if idx not in idxs or kind != "cover": continue cov = cdcrop.analyze_cover(arr, rect) comp = cdcrop.angle_components(gray, cov, inset) losses, blurs = [], [] for v in variants: saved = {k: getattr(cdcrop, k) for k in v} for k, val in v.items(): setattr(cdcrop, k, val) losses.append(cdcrop.component_loss(comp, gray)) blurs.append(comp["blur"]) for k, val in saved.items(): setattr(cdcrop, k, val) comp["losses"], comp["blurs"] = losses, blurs # sanity check: do our edge angles match what the interface logged? mism = [] for k, v in picks[cid]["logged"].items(): if v not in (None, "", "soft") and k in comp["named"]: if abs(float(v) - comp["named"][k]) > 0.01: mism.append(f"{k} {float(v):+.3f} logged vs {comp['named'][k]:+.3f} now") covers[cid] = comp note = (" !! differs from the logged run: " + "; ".join(mism)) if mism else "" print(f" {cid}: measured{note}") print(f" ({base}: {time.time() - t0:.0f}s)") return covers, variants def evaluate(covers, picks, variants, tol): """Score every weighting: how many picks it reproduces, and how closely.""" names = list(WEIGHT_GRID) grid = list(itertools.product(range(len(variants)), *[range(len(v)) for v in WEIGHT_GRID.values()])) results = [] cids = sorted(covers) for gi in grid: vi, wi = gi[0], gi[1:] W = {n: WEIGHT_GRID[n][j] for n, j in zip(names, wi)} errs = [] for cid in cids: c = covers[cid] total = (c["losses"][vi] + W["W_V"] * c["conf_v"] * c["cv"] + W["W_V"] * H_RATIO * c["conf_h"] * c["chh"] + W["W_LEFT"] * c["left_px"] + W["W_EDGE"] * c["edge_px"] + W["W_BLUR"] * c["blurs"][vi]) best = c["thetas"][int(np.argmin(total))] errs.append(abs(best - picks[cid]["chosen"]) * math.pi / 180 * c["side"]) errs = np.array(errs) results.append((gi, int((errs <= tol).sum()), float(np.minimum(errs, 20).sum()), errs)) return results, cids def robust_best(results, mask=None, tol=2.0): """Most picks reproduced (only covers in mask, if given); among those, the setting whose grid neighbours do as well (the middle of a good region), then the smallest total error.""" sel = (lambda e: e) if mask is None else (lambda e: e[mask]) score = {r[0]: int((sel(r[3]) <= tol).sum()) for r in results} err = {r[0]: float(np.minimum(sel(r[3]), 20).sum()) for r in results} top = max(score.values()) def support(gi): n = 0 for d in range(len(gi)): for s in (-1, 1): nb = list(gi); nb[d] += s if score.get(tuple(nb)) == top: n += 1 return n cands = [r for r in results if score[r[0]] == top] return max(cands, key=lambda r: (support(r[0]), -err[r[0]])), top def leave_one_out(results, n, tol): """Tune on all covers but one, check the one left out; repeat for each. The honest measure of whether tuning learned your taste or just fit these particular covers.""" hits = 0 for k in range(n): mask = np.ones(n, bool); mask[k] = False (r, _) = robust_best(results, mask, tol) hits += int(r[3][k] <= tol) return hits def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--scans", required=True, help="folder with the original scan BMPs") ap.add_argument("--decisions", required=True, help="cdcrop_decisions.csv from the interface's output folder") ap.add_argument("--tol", type=float, default=2.0, help="px of drift across the cover that counts as a match") ap.add_argument("--inset", type=float, default=1.0) ap.add_argument("--no-calibration", action="store_true", help="tune without the scanner calibration") ap.add_argument("--dry-run", action="store_true", help="report only; don't save cdcrop_tuning.json") ap.add_argument("--force", action="store_true", help="save the tuned weights even if they don't beat the current ones on unseen covers") a = ap.parse_args() warnings.simplefilter("ignore") cal_path = os.path.join(os.path.dirname(os.path.abspath(cdcrop.__file__)), "scanner_calibration.json") cal = None if not a.no_calibration and os.path.exists(cal_path): with open(cal_path) as fh: cal = json.load(fh) print(f"scanner calibration: stretch {cal['y_stretch_pct']:+.3f}%, shear {cal['shear_deg']:+.4f} deg") else: print("no scanner calibration in use") picks = read_picks(a.decisions) print(f"{len(picks)} reviewed covers in {os.path.basename(a.decisions)}; measuring them…") covers, variants = analyse(a.scans, picks, a.inset, cal) if len(covers) < 3: print("!! need at least 3 reviewed covers to tune"); return results, cids = evaluate(covers, picks, variants, a.tol) # the current settings, for comparison cur = {k: getattr(cdcrop, k) for k in cdcrop.TUNABLE} cur_errs = [] for cid in cids: c = covers[cid] ch = cdcrop.combine_angle(c) # uses the loss computed at current detail settings cur_errs.append(abs(ch["theta"] - picks[cid]["chosen"]) * math.pi / 180 * c["side"]) cur_errs = np.array(cur_errs) (gi, n, tot, errs), top = robust_best(results, None, a.tol) cur_hits = int((cur_errs <= a.tol).sum()) loo = leave_one_out(results, len(cids), a.tol) W = {"W_V": WEIGHT_GRID["W_V"][gi[1]], "W_LEFT": WEIGHT_GRID["W_LEFT"][gi[2]], "W_EDGE": WEIGHT_GRID["W_EDGE"][gi[3]], "W_BLUR": WEIGHT_GRID["W_BLUR"][gi[4]]} W["W_H"] = W["W_V"] * H_RATIO W.update(variants[gi[0]]) print(f"\ncurrent settings reproduce {int((cur_errs <= a.tol).sum())} of {len(cids)} picks; " f"tuned settings reproduce {n} of {len(cids)}") print(f"{'cover':22s} {'your pick':>10s} {'current':>9s} {'tuned':>9s} (px of drift from your pick)") for cid, e_now, e_new in zip(cids, cur_errs, errs): c = covers[cid] print(f"{cid:22s} {picks[cid]['chosen']:+10.3f} {e_now:8.1f}px {e_new:8.1f}px" + (" <- still off" if e_new > a.tol else "")) print("\ntuned weights: " + ", ".join(f"{k}={v:g}" for k, v in W.items())) print(f"check on unseen covers (leave-one-out): tuned settings predict {loo} of {len(cids)}, " f"current settings {cur_hits} of {len(cids)}") if loo <= cur_hits and not a.force: print("-> tuning doesn't predict unseen covers any better than the current settings yet, so it was NOT " "saved (it would only be fitting these particular covers). Review more covers and run this again; " "use --force to save anyway.") return if a.dry_run: return W = {**cur, **W} # keep any tunables this script doesn't touch (e.g. CORNER_LENIENCY, # fit by cdcrop_tune_crop.py) at their current value instead of dropping them out = {"weights": W, "picks": len(cids), "reproduced": n, "tolerance_px": a.tol, "calibrated": cal is not None, "date": time.strftime("%Y-%m-%d %H:%M"), "previous": cur} if os.path.exists(cdcrop.TUNING_PATH): os.replace(cdcrop.TUNING_PATH, cdcrop.TUNING_PATH + ".bak") with open(cdcrop.TUNING_PATH, "w") as fh: json.dump(out, fh, indent=2) print(f"saved to {cdcrop.TUNING_PATH} (previous version kept as .bak). Re-process to apply.") if __name__ == "__main__": main()