From 5549217db59315b1616451f596f0b17720da29d2 Mon Sep 17 00:00:00 2001 From: bob walker Date: Thu, 10 Sep 2026 12:07:45 +0100 Subject: [PATCH] Fetch every wiki photo at its largest size into the backup directory Walk the grouped index, ask flickr for each new photo, download the best size that actually serves, and write the sidecar only once the file is renamed into place. Photos already complete cost no flickr calls; a change in which pages use one rewrites just the sidecar. Every failure is recorded with the pages affected and the run carries on. Co-Authored-By: Claude Fable 5.1 --- rgl-photo-backup | 140 ++++++++++++++++++++++++++++ test_rgl_photo_backup.py | 196 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+) diff --git a/rgl-photo-backup b/rgl-photo-backup index 9ae6965..c95c620 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -779,5 +779,145 @@ def tagsfor(side): return tags +class Failures: + def __init__(self): + self.items = [] + + def add(self, key, nodes, reason): + names = [n.get("name", n.get("param", "")) for n in nodes] + self.items.append((key, names, str(reason))) + STATS["failed"] += 1 + warn("%s (%s): %s" % (key, ", ".join(names) or "no pages", reason)) + + def __len__(self): + return len(self.items) + + +def nodeentry(node): + return {k: node.get(k, "") for k in ("param", "name", "url", "lat", "long")} + +# By node name, so a sidecar and the description built from it do not change +# just because the index listed the same pages in another order. +def nodeentries(nodes): + return [nodeentry(n) for n in sorted(nodes, key=lambda n: n.get("param", ""))] + +# The pages a sidecar records, for naming a failure. A sidecar on disk can be +# any shape at all, and the failure path must not itself raise. +def sidenodes(side): + rgl = side.get("rgl") + nodes = rgl.get("nodes") if isinstance(rgl, dict) else None + return [n for n in nodes if isinstance(n, dict)] if isinstance(nodes, list) else [] + +def buildsidecar(group, info, pick, filename): + side = { + "key": group["key"], + "file": filename, + "source": group["source"], + "rgl": {"copyright": group["copyright"], "licence": group["licence"], + "nodes": nodeentries(group["nodes"])}, + "fetched": utcnow(), + } + if group["source"] == "flickr": + side["flickr"] = dict(info, size_label=pick["label"], + width=pick["width"], height=pick["height"]) + else: + side["page"] = group["page"] or group["image"] + return side + +# +# Try each candidate url in turn, best first; a 404 on one size means try the +# next, anything else is a real failure. Returns (candidate, extension). +# +def downloadany(candidates, partial, context): + last = None + for pick in candidates: + try: + ctype, written = download(pick["source"], context, partial, pace=True) + except HTTPFailure as e: + if e.code != 404: + raise + last = e + continue + if written == 0: + last = BackupError("%s: empty response" % context) + continue + return (pick, extensionfor(pick["source"], ctype)) + raise last or BackupError("%s: no downloadable size" % context) + +def fetchphoto(directory, group, flickrkey, dryrun=False): + key = group["key"] + context = "photo %s" % key + sidepath = sidecarpath(directory, key) + + if iscomplete(directory, key): + held = readsidecar(sidepath) + if not isinstance(held.get("rgl"), dict): + held["rgl"] = {} + wanted = nodeentries(group["nodes"]) + if held["rgl"].get("nodes") == wanted and held["rgl"].get("copyright") == group["copyright"] \ + and held["rgl"].get("licence") == group["licence"]: + STATS["skipped"] += 1 + return (held, "skipped") + held["rgl"].update(nodes=wanted, copyright=group["copyright"], licence=group["licence"]) + if not dryrun: + writejson(sidepath, held) + STATS["rewritten"] += 1 + return (held, "updated") + + if dryrun: + return (None, "would fetch") + + if group["source"] == "flickr": + info = getinfo(group["id"], flickrkey) + candidates = picksizes(getsizes(group["id"], flickrkey)) + if not candidates: + raise BackupError("%s: flickr lists no downloadable sizes" % context) + else: + info = None + candidates = [{"label": "", "width": 0, "height": 0, "source": group["image"], "media": "photo"}] + + partial = os.path.join(directory, key + ".part") + try: + pick, ext = downloadany(candidates, partial, context) + filename = key + ext + side = buildsidecar(group, info, pick, filename) + os.replace(partial, os.path.join(directory, filename)) + writejson(sidepath, side) + finally: + if os.path.exists(partial): + os.remove(partial) + STATS["fetched"] += 1 + return (side, "fetched") + +def fetchphase(conf, flickrkey): + failures = Failures() + if not conf["dry_run"]: + os.makedirs(conf["directory"], exist_ok=True) + nodes = loadindex(conf["index_url"]) + groups, rejected = groupphotos(nodes) + say("index: %d nodes, %d photos on %d of them" + % (len(nodes), len(groups), sum(len(g["nodes"]) for g in groups.values()))) + for node, reason in rejected: + failures.add(node["param"], [node], reason) + + done = 0 + for key in sorted(groups): + if conf["limit"] is not None and done >= conf["limit"]: + break + group = groups[key] + try: + side, action = fetchphoto(conf["directory"], group, flickrkey, conf["dry_run"]) + except BackupError as e: + failures.add(key, group["nodes"], e) + continue + except OSError as e: + failures.add(key, group["nodes"], "%s: %s" % (type(e).__name__, e)) + continue + if action != "skipped": + done += 1 + say("%s: %s (%s)" % (key, action, ", ".join(n["name"] for n in group["nodes"]))) + return failures + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index 177e311..70d13fc 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -589,6 +589,14 @@ class Sidecars(TestCase): self.assertTrue(rb.iscomplete(self.dir, "111")) self.write("111.part", b"x") self.assertFalse(rb.iscomplete(self.dir, "111")) # interrupted refetch + + def test_sidenodes_survives_any_shape(self): + self.assertEqual(rb.sidenodes({}), []) + self.assertEqual(rb.sidenodes({"rgl": "nonsense"}), []) + self.assertEqual(rb.sidenodes({"rgl": {"nodes": "nonsense"}}), []) + self.assertEqual(rb.sidenodes({"rgl": {"nodes": ["nonsense"]}}), []) + self.assertEqual(rb.sidenodes(sidecar())[0]["name"], "A, E1") + def test_listsidecars_unreadable_directory_is_an_error(self): with self.assertRaises(rb.BackupError): rb.listsidecars(os.path.join(self.dir, "nowhere")) @@ -736,3 +744,191 @@ class Tags(TestCase): def test_photographer_whitespace_and_slashes(self): side = sidecar(copyright=" Some / One ") self.assertIn("rgl/photographer/Some - One", rb.tagsfor(side)) + + +class FetchPhoto(TestCase): + def group(self, key="111", nodes=None): + nodes = nodes or [{"param": "A,_E1", "name": "A, E1", + "url": "https://london.randomness.org.uk/wiki.cgi?A%2C_E1", + "image": "https://farm1.static.flickr.com/1/111_a_m.jpg", + "page": "https://www.flickr.com/photos/x/111/", + "copyright": "P", "licence": "https://creativecommons.org/licenses/by-sa/2.0/", + "lat": "51.5", "long": "-0.1"}] + return {"key": key, "source": "flickr", "id": key, "image": nodes[0]["image"], + "page": nodes[0]["page"], "copyright": "P", + "licence": "https://creativecommons.org/licenses/by-sa/2.0/", "nodes": nodes} + + def test_fetches_largest_and_writes_sidecar(self): + rb.OPEN = FakeOpen( + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Small", 240, 180, "https://x/111_s.jpg"), + ("Original", 800, 600, "https://x/111_o.png"))), + FakeResponse(b"PNGDATA", headers={"Content-Type": "image/png"})) + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "fetched") + self.assertEqual(side["file"], "111.png") + self.assertEqual(side["flickr"]["size_label"], "Original") + self.assertEqual(side["flickr"]["width"], 800) + self.assertEqual(side["rgl"]["nodes"][0]["param"], "A,_E1") + self.assertNotIn("image", side["rgl"]["nodes"][0]) + self.assertTrue(rb.iscomplete(self.dir, "111")) + self.assertEqual(open(os.path.join(self.dir, "111.png"), "rb").read(), b"PNGDATA") + self.assertFalse(os.path.exists(os.path.join(self.dir, "111.part"))) + + def test_falls_back_when_largest_404s(self): + rb.OPEN = FakeOpen( + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Large", 1024, 768, "https://x/111_b.jpg"), + ("Original", 800, 600, "https://x/111_o.jpg"))), + http_error(404), + FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"})) + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(side["flickr"]["size_label"], "Large") + self.assertIn("111_o.jpg", rb.OPEN.requests[2].full_url) + + def test_gone_photo_raises_and_writes_nothing(self): + rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"})) + with self.assertRaises(rb.PhotoGone): + rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(os.listdir(self.dir), []) + + def test_complete_photo_is_skipped_without_calls(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + rb.OPEN = FakeOpen() + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "skipped") + + def test_changed_nodes_rewrite_sidecar_only(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + rb.OPEN = FakeOpen() + g = self.group() + g["nodes"].append(dict(g["nodes"][0], param="B,_E2", name="B, E2", url="https://x/B")) + side, action = rb.fetchphoto(self.dir, g, "k") + self.assertEqual(action, "updated") + self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 2) + self.assertEqual(rb.STATS["rewritten"], 1) + + def test_nodes_are_sorted_by_param(self): + rb.OPEN = FakeOpen( + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))), + FakeResponse(b"J", headers={"Content-Type": "image/jpeg"})) + g = self.group() + g["nodes"].insert(0, dict(g["nodes"][0], param="Z,_E9", name="Z, E9")) + side, action = rb.fetchphoto(self.dir, g, "k") + self.assertEqual([n["param"] for n in side["rgl"]["nodes"]], ["A,_E1", "Z,_E9"]) + # and the same order the comparison uses, so a re-run rewrites nothing + self.assertEqual(rb.fetchphoto(self.dir, g, "k")[1], "skipped") + + def test_downloadany_skips_an_empty_body(self): + rb.OPEN = FakeOpen(FakeResponse(b""), + FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"})) + candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg", + "media": "photo"}, + {"label": "Large", "width": 4, "height": 3, "source": "https://x/l.jpg", + "media": "photo"}] + partial = os.path.join(self.dir, "111.part") + pick, ext = rb.downloadany(candidates, partial, "photo 111") + self.assertEqual((pick["label"], ext), ("Large", ".jpg")) + + def test_downloadany_with_nothing_but_empty_bodies_fails(self): + rb.OPEN = FakeOpen(FakeResponse(b"")) + candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg", + "media": "photo"}] + with self.assertRaises(rb.BackupError) as cm: + rb.downloadany(candidates, os.path.join(self.dir, "111.part"), "photo 111") + self.assertIn("empty", str(cm.exception)) + + def test_partial_file_is_refetched(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + self.write("111.part", b"half") + rb.OPEN = FakeOpen(FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))), + FakeResponse(b"FULL", headers={"Content-Type": "image/jpeg"})) + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "fetched") + self.assertEqual(open(os.path.join(self.dir, "111.jpg"), "rb").read(), b"FULL") + + def test_sidecar_without_an_rgl_block_is_rewritten_not_fatal(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), {"key": "111", "file": "111.jpg"}) + self.write("111.jpg", b"x") + rb.OPEN = FakeOpen() + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "updated") + self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 1) + + def test_dry_run_calls_nothing(self): + rb.OPEN = FakeOpen() + side, action = rb.fetchphoto(self.dir, self.group(), "k", dryrun=True) + self.assertEqual(action, "would fetch") + self.assertEqual(os.listdir(self.dir), []) + + def test_non_flickr_photo_downloads_image_url(self): + g = self.group(key="url-0123456789abcdef") + g["source"] = "url" + del g["id"] + g["image"] = g["nodes"][0]["image"] = "https://www.geograph.org.uk/p/1.jpg" + g["page"] = "https://www.geograph.org.uk/photo/1" + rb.OPEN = FakeOpen(FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"})) + side, action = rb.fetchphoto(self.dir, g, "k") + self.assertEqual(side["source"], "url") + self.assertEqual(side["page"], "https://www.geograph.org.uk/photo/1") + self.assertNotIn("flickr", side) + self.assertEqual(side["file"], "url-0123456789abcdef.jpg") + + +class FetchPhase(TestCase): + def conf(self, **over): + conf = {"directory": self.dir, "index_url": "http://t/index", "limit": None, + "dry_run": False, "quiet": False} + conf.update(over) + return conf + + def test_runs_over_index_and_collects_failures(self): + index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg", + page="https://www.flickr.com/photos/x/111/"), + index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg", + page="https://www.flickr.com/photos/x/222/"), + index_entry("C,_E3", "junk")] + rb.OPEN = FakeOpen( + FakeResponse(json.dumps(index).encode()), + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))), + FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}), + FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"})) + failures = rb.fetchphase(self.conf(), "k") + self.assertTrue(rb.iscomplete(self.dir, "111")) + self.assertEqual(rb.STATS["fetched"], 1) + self.assertEqual(len(failures), 2) + reasons = [f[2] for f in failures.items] + self.assertTrue(any("Photo not found" in r for r in reasons), reasons) + self.assertTrue(any("not a url" in r for r in reasons), reasons) + names = [n for f in failures.items for n in f[1]] + self.assertIn("B, E2", names) + + def test_limit_stops_early(self): + index = [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"), + index_entry("B", "https://farm1.static.flickr.com/1/222_b_m.jpg")] + rb.OPEN = FakeOpen( + FakeResponse(json.dumps(index).encode()), + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))), + FakeResponse(b"J", headers={"Content-Type": "image/jpeg"})) + rb.fetchphase(self.conf(limit=1), "k") + self.assertEqual(rb.STATS["fetched"], 1) + self.assertFalse(os.path.exists(rb.sidecarpath(self.dir, "222"))) + + def test_dry_run_creates_no_directory(self): + room = os.path.join(self.dir, "not-yet") + rb.OPEN = FakeOpen(FakeResponse(json.dumps( + [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg")]).encode())) + rb.fetchphase(self.conf(directory=room, dry_run=True), "k") + self.assertFalse(os.path.exists(room)) + + def test_index_failure_is_fatal(self): + rb.OPEN = FakeOpen(http_error(404)) + with self.assertRaises(rb.BackupError): + rb.fetchphase(self.conf(), "k")