234 lines
11 KiB
Python
234 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
cdcrop_tune.py - tune cdcrop's automatic angle choice to your own picks.
|
|
|
|
python cdcrop_tune.py --scans "C:\\path\\to\\scans" --decisions "C:\\path\\to\\output\\cdcrop_decisions.csv"
|
|
|
|
Runs on your computer with your scans, so the result matches exactly what the
|
|
cropper does there. For every cover you reviewed in the interface, it
|
|
measures everything the automatic choice weighs, then tries a few thousand
|
|
weightings and keeps the one that reproduces your picks best - preferring
|
|
settings in the middle of a good region, so small differences on new covers
|
|
don't flip results. The winner is saved to cdcrop_tuning.json next to
|
|
cdcrop.py and used automatically from then on (delete that file to go back
|
|
to the built-in weights). Run it again whenever you've reviewed a new batch.
|
|
|
|
A pick counts as reproduced when the automatic angle lands within --tol px
|
|
(default 2) of yours, measured as drift across the whole cover. New weights
|
|
are only saved if they predict covers they weren't tuned on (leave-one-out)
|
|
better than the current ones - otherwise they'd just be memorising.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import itertools
|
|
import json
|
|
import math
|
|
import os
|
|
import sys
|
|
import time
|
|
import warnings
|
|
|
|
import numpy as np
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import cdcrop # noqa: E402
|
|
|
|
DETAIL_GRID = {"DEEP_BLEND": (0.0, 0.5, 1.0), "SAL_POWER": (1.0, 2.0), "SAL_FLOOR": (50, 60, 70)}
|
|
WEIGHT_GRID = {"W_V": (0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 4.0, 6.0, 8.0),
|
|
"W_LEFT": (0.0, 0.05, 0.1, 0.2, 0.4),
|
|
"W_EDGE": (0.0, 0.01, 0.02, 0.05),
|
|
"W_BLUR": (0.0, 0.25, 0.5)}
|
|
H_RATIO = 0.5 # W_H = W_V * H_RATIO (horizontal lines count half, as calibrated so far)
|
|
|
|
|
|
def read_picks(path):
|
|
"""Latest reviewed angle per cover (discs are skipped - they're tuned by hand)."""
|
|
picks = {}
|
|
with open(path, newline="") as fh:
|
|
for row in csv.DictReader(fh):
|
|
cands = json.loads(row.get("candidates") or "{}")
|
|
if "as scanned" in cands: # a disc
|
|
continue
|
|
picks[row["cover"]] = {"chosen": float(row["chosen_deg"]), "how": row["how"],
|
|
"logged": {k: row.get(f"{k}_deg") for k in ("left", "right", "top", "bottom")}}
|
|
return picks
|
|
|
|
|
|
def analyse(scans_dir, picks, inset, cal):
|
|
"""angle_components for every reviewed cover, with one loss per detail variant."""
|
|
by_scan = {}
|
|
for cid in picks:
|
|
base, idx = cid.rsplit("_", 1)
|
|
by_scan.setdefault(base, []).append(int(idx))
|
|
variants = [dict(zip(DETAIL_GRID, v)) for v in itertools.product(*DETAIL_GRID.values())]
|
|
covers = {}
|
|
for base, idxs in sorted(by_scan.items()):
|
|
path = next((os.path.join(scans_dir, f) for f in os.listdir(scans_dir)
|
|
if os.path.splitext(f)[0] == base and f.lower().endswith(".bmp")), None)
|
|
if not path:
|
|
print(f" !! {base}.bmp not found in the scans folder - skipping its covers")
|
|
continue
|
|
t0 = time.time()
|
|
arr8, _, dpi = cdcrop.load_scan(path)
|
|
arr, _ = cdcrop.true_space(arr8.astype(np.float32), cal)
|
|
gray = arr[..., :3].mean(axis=2) if arr.ndim == 3 else arr
|
|
objs = cdcrop.find_objects(arr, max(max(idxs), 2), dpi)
|
|
for idx, (kind, rect, _) in enumerate(objs, start=1):
|
|
cid = f"{base}_{idx}"
|
|
if idx not in idxs or kind != "cover":
|
|
continue
|
|
cov = cdcrop.analyze_cover(arr, rect)
|
|
comp = cdcrop.angle_components(gray, cov, inset)
|
|
losses, blurs = [], []
|
|
for v in variants:
|
|
saved = {k: getattr(cdcrop, k) for k in v}
|
|
for k, val in v.items():
|
|
setattr(cdcrop, k, val)
|
|
losses.append(cdcrop.component_loss(comp, gray))
|
|
blurs.append(comp["blur"])
|
|
for k, val in saved.items():
|
|
setattr(cdcrop, k, val)
|
|
comp["losses"], comp["blurs"] = losses, blurs
|
|
# sanity check: do our edge angles match what the interface logged?
|
|
mism = []
|
|
for k, v in picks[cid]["logged"].items():
|
|
if v not in (None, "", "soft") and k in comp["named"]:
|
|
if abs(float(v) - comp["named"][k]) > 0.01:
|
|
mism.append(f"{k} {float(v):+.3f} logged vs {comp['named'][k]:+.3f} now")
|
|
covers[cid] = comp
|
|
note = (" !! differs from the logged run: " + "; ".join(mism)) if mism else ""
|
|
print(f" {cid}: measured{note}")
|
|
print(f" ({base}: {time.time() - t0:.0f}s)")
|
|
return covers, variants
|
|
|
|
|
|
def evaluate(covers, picks, variants, tol):
|
|
"""Score every weighting: how many picks it reproduces, and how closely."""
|
|
names = list(WEIGHT_GRID)
|
|
grid = list(itertools.product(range(len(variants)), *[range(len(v)) for v in WEIGHT_GRID.values()]))
|
|
results = []
|
|
cids = sorted(covers)
|
|
for gi in grid:
|
|
vi, wi = gi[0], gi[1:]
|
|
W = {n: WEIGHT_GRID[n][j] for n, j in zip(names, wi)}
|
|
errs = []
|
|
for cid in cids:
|
|
c = covers[cid]
|
|
total = (c["losses"][vi] + W["W_V"] * c["conf_v"] * c["cv"]
|
|
+ W["W_V"] * H_RATIO * c["conf_h"] * c["chh"]
|
|
+ W["W_LEFT"] * c["left_px"] + W["W_EDGE"] * c["edge_px"]
|
|
+ W["W_BLUR"] * c["blurs"][vi])
|
|
best = c["thetas"][int(np.argmin(total))]
|
|
errs.append(abs(best - picks[cid]["chosen"]) * math.pi / 180 * c["side"])
|
|
errs = np.array(errs)
|
|
results.append((gi, int((errs <= tol).sum()), float(np.minimum(errs, 20).sum()), errs))
|
|
return results, cids
|
|
|
|
|
|
def robust_best(results, mask=None, tol=2.0):
|
|
"""Most picks reproduced (only covers in mask, if given); among those, the
|
|
setting whose grid neighbours do as well (the middle of a good region),
|
|
then the smallest total error."""
|
|
sel = (lambda e: e) if mask is None else (lambda e: e[mask])
|
|
score = {r[0]: int((sel(r[3]) <= tol).sum()) for r in results}
|
|
err = {r[0]: float(np.minimum(sel(r[3]), 20).sum()) for r in results}
|
|
top = max(score.values())
|
|
def support(gi):
|
|
n = 0
|
|
for d in range(len(gi)):
|
|
for s in (-1, 1):
|
|
nb = list(gi); nb[d] += s
|
|
if score.get(tuple(nb)) == top:
|
|
n += 1
|
|
return n
|
|
cands = [r for r in results if score[r[0]] == top]
|
|
return max(cands, key=lambda r: (support(r[0]), -err[r[0]])), top
|
|
|
|
|
|
def leave_one_out(results, n, tol):
|
|
"""Tune on all covers but one, check the one left out; repeat for each.
|
|
The honest measure of whether tuning learned your taste or just fit
|
|
these particular covers."""
|
|
hits = 0
|
|
for k in range(n):
|
|
mask = np.ones(n, bool); mask[k] = False
|
|
(r, _) = robust_best(results, mask, tol)
|
|
hits += int(r[3][k] <= tol)
|
|
return hits
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--scans", required=True, help="folder with the original scan BMPs")
|
|
ap.add_argument("--decisions", required=True, help="cdcrop_decisions.csv from the interface's output folder")
|
|
ap.add_argument("--tol", type=float, default=2.0, help="px of drift across the cover that counts as a match")
|
|
ap.add_argument("--inset", type=float, default=1.0)
|
|
ap.add_argument("--no-calibration", action="store_true", help="tune without the scanner calibration")
|
|
ap.add_argument("--dry-run", action="store_true", help="report only; don't save cdcrop_tuning.json")
|
|
ap.add_argument("--force", action="store_true", help="save the tuned weights even if they don't beat the current ones on unseen covers")
|
|
a = ap.parse_args()
|
|
warnings.simplefilter("ignore")
|
|
|
|
cal_path = os.path.join(os.path.dirname(os.path.abspath(cdcrop.__file__)), "scanner_calibration.json")
|
|
cal = None
|
|
if not a.no_calibration and os.path.exists(cal_path):
|
|
with open(cal_path) as fh:
|
|
cal = json.load(fh)
|
|
print(f"scanner calibration: stretch {cal['y_stretch_pct']:+.3f}%, shear {cal['shear_deg']:+.4f} deg")
|
|
else:
|
|
print("no scanner calibration in use")
|
|
picks = read_picks(a.decisions)
|
|
print(f"{len(picks)} reviewed covers in {os.path.basename(a.decisions)}; measuring them…")
|
|
covers, variants = analyse(a.scans, picks, a.inset, cal)
|
|
if len(covers) < 3:
|
|
print("!! need at least 3 reviewed covers to tune"); return
|
|
|
|
results, cids = evaluate(covers, picks, variants, a.tol)
|
|
# the current settings, for comparison
|
|
cur = {k: getattr(cdcrop, k) for k in cdcrop.TUNABLE}
|
|
cur_errs = []
|
|
for cid in cids:
|
|
c = covers[cid]
|
|
ch = cdcrop.combine_angle(c) # uses the loss computed at current detail settings
|
|
cur_errs.append(abs(ch["theta"] - picks[cid]["chosen"]) * math.pi / 180 * c["side"])
|
|
cur_errs = np.array(cur_errs)
|
|
(gi, n, tot, errs), top = robust_best(results, None, a.tol)
|
|
cur_hits = int((cur_errs <= a.tol).sum())
|
|
loo = leave_one_out(results, len(cids), a.tol)
|
|
W = {"W_V": WEIGHT_GRID["W_V"][gi[1]], "W_LEFT": WEIGHT_GRID["W_LEFT"][gi[2]], "W_EDGE": WEIGHT_GRID["W_EDGE"][gi[3]],
|
|
"W_BLUR": WEIGHT_GRID["W_BLUR"][gi[4]]}
|
|
W["W_H"] = W["W_V"] * H_RATIO
|
|
W.update(variants[gi[0]])
|
|
|
|
print(f"\ncurrent settings reproduce {int((cur_errs <= a.tol).sum())} of {len(cids)} picks; "
|
|
f"tuned settings reproduce {n} of {len(cids)}")
|
|
print(f"{'cover':22s} {'your pick':>10s} {'current':>9s} {'tuned':>9s} (px of drift from your pick)")
|
|
for cid, e_now, e_new in zip(cids, cur_errs, errs):
|
|
c = covers[cid]
|
|
print(f"{cid:22s} {picks[cid]['chosen']:+10.3f} {e_now:8.1f}px {e_new:8.1f}px"
|
|
+ (" <- still off" if e_new > a.tol else ""))
|
|
print("\ntuned weights: " + ", ".join(f"{k}={v:g}" for k, v in W.items()))
|
|
print(f"check on unseen covers (leave-one-out): tuned settings predict {loo} of {len(cids)}, "
|
|
f"current settings {cur_hits} of {len(cids)}")
|
|
if loo <= cur_hits and not a.force:
|
|
print("-> tuning doesn't predict unseen covers any better than the current settings yet, so it was NOT "
|
|
"saved (it would only be fitting these particular covers). Review more covers and run this again; "
|
|
"use --force to save anyway.")
|
|
return
|
|
if a.dry_run:
|
|
return
|
|
W = {**cur, **W} # keep any tunables this script doesn't touch (e.g. CORNER_LENIENCY,
|
|
# fit by cdcrop_tune_crop.py) at their current value instead of dropping them
|
|
out = {"weights": W, "picks": len(cids), "reproduced": n, "tolerance_px": a.tol,
|
|
"calibrated": cal is not None, "date": time.strftime("%Y-%m-%d %H:%M"),
|
|
"previous": cur}
|
|
if os.path.exists(cdcrop.TUNING_PATH):
|
|
os.replace(cdcrop.TUNING_PATH, cdcrop.TUNING_PATH + ".bak")
|
|
with open(cdcrop.TUNING_PATH, "w") as fh:
|
|
json.dump(out, fh, indent=2)
|
|
print(f"saved to {cdcrop.TUNING_PATH} (previous version kept as .bak). Re-process to apply.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|