Fetch every wiki photo at its largest size into the backup directory

Walk the grouped index, ask flickr for each new photo, download the best
size that actually serves, and write the sidecar only once the file is
renamed into place. Photos already complete cost no flickr calls; a change
in which pages use one rewrites just the sidecar. Every failure is
recorded with the pages affected and the run carries on.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

rgl-photo-backup

@@ -779,5 +779,145 @@ def tagsfor(side):
     return tags
 
 
+class Failures:
+    def __init__(self):
+        self.items = []
+
+    def add(self, key, nodes, reason):
+        names = [n.get("name", n.get("param", "")) for n in nodes]
+        self.items.append((key, names, str(reason)))
+        STATS["failed"] += 1
+        warn("%s (%s): %s" % (key, ", ".join(names) or "no pages", reason))
+
+    def __len__(self):
+        return len(self.items)
+
+
+def nodeentry(node):
+    return {k: node.get(k, "") for k in ("param", "name", "url", "lat", "long")}
+
+# By node name, so a sidecar and the description built from it do not change
+# just because the index listed the same pages in another order.
+def nodeentries(nodes):
+    return [nodeentry(n) for n in sorted(nodes, key=lambda n: n.get("param", ""))]
+
+# The pages a sidecar records, for naming a failure. A sidecar on disk can be
+# any shape at all, and the failure path must not itself raise.
+def sidenodes(side):
+    rgl = side.get("rgl")
+    nodes = rgl.get("nodes") if isinstance(rgl, dict) else None
+    return [n for n in nodes if isinstance(n, dict)] if isinstance(nodes, list) else []
+
+def buildsidecar(group, info, pick, filename):
+    side = {
+        "key":     group["key"],
+        "file":    filename,
+        "source":  group["source"],
+        "rgl":     {"copyright": group["copyright"], "licence": group["licence"],
+                    "nodes": nodeentries(group["nodes"])},
+        "fetched": utcnow(),
+    }
+    if group["source"] == "flickr":
+        side["flickr"] = dict(info, size_label=pick["label"],
+                              width=pick["width"], height=pick["height"])
+    else:
+        side["page"] = group["page"] or group["image"]
+    return side
+
+#
+# Try each candidate url in turn, best first; a 404 on one size means try the
+# next, anything else is a real failure. Returns (candidate, extension).
+#
+def downloadany(candidates, partial, context):
+    last = None
+    for pick in candidates:
+        try:
+            ctype, written = download(pick["source"], context, partial, pace=True)
+        except HTTPFailure as e:
+            if e.code != 404:
+                raise
+            last = e
+            continue
+        if written == 0:
+            last = BackupError("%s: empty response" % context)
+            continue
+        return (pick, extensionfor(pick["source"], ctype))
+    raise last or BackupError("%s: no downloadable size" % context)
+
+def fetchphoto(directory, group, flickrkey, dryrun=False):
+    key = group["key"]
+    context = "photo %s" % key
+    sidepath = sidecarpath(directory, key)
+
+    if iscomplete(directory, key):
+        held = readsidecar(sidepath)
+        if not isinstance(held.get("rgl"), dict):
+            held["rgl"] = {}
+        wanted = nodeentries(group["nodes"])
+        if held["rgl"].get("nodes") == wanted and held["rgl"].get("copyright") == group["copyright"] \
+                and held["rgl"].get("licence") == group["licence"]:
+            STATS["skipped"] += 1
+            return (held, "skipped")
+        held["rgl"].update(nodes=wanted, copyright=group["copyright"], licence=group["licence"])
+        if not dryrun:
+            writejson(sidepath, held)
+            STATS["rewritten"] += 1
+        return (held, "updated")
+
+    if dryrun:
+        return (None, "would fetch")
+
+    if group["source"] == "flickr":
+        info = getinfo(group["id"], flickrkey)
+        candidates = picksizes(getsizes(group["id"], flickrkey))
+        if not candidates:
+            raise BackupError("%s: flickr lists no downloadable sizes" % context)
+    else:
+        info = None
+        candidates = [{"label": "", "width": 0, "height": 0, "source": group["image"], "media": "photo"}]
+
+    partial = os.path.join(directory, key + ".part")
+    try:
+        pick, ext = downloadany(candidates, partial, context)
+        filename = key + ext
+        side = buildsidecar(group, info, pick, filename)
+        os.replace(partial, os.path.join(directory, filename))
+        writejson(sidepath, side)
+    finally:
+        if os.path.exists(partial):
+            os.remove(partial)
+    STATS["fetched"] += 1
+    return (side, "fetched")
+
+def fetchphase(conf, flickrkey):
+    failures = Failures()
+    if not conf["dry_run"]:
+        os.makedirs(conf["directory"], exist_ok=True)
+    nodes = loadindex(conf["index_url"])
+    groups, rejected = groupphotos(nodes)
+    say("index: %d nodes, %d photos on %d of them"
+        % (len(nodes), len(groups), sum(len(g["nodes"]) for g in groups.values())))
+    for node, reason in rejected:
+        failures.add(node["param"], [node], reason)
+
+    done = 0
+    for key in sorted(groups):
+        if conf["limit"] is not None and done >= conf["limit"]:
+            break
+        group = groups[key]
+        try:
+            side, action = fetchphoto(conf["directory"], group, flickrkey, conf["dry_run"])
+        except BackupError as e:
+            failures.add(key, group["nodes"], e)
+            continue
+        except OSError as e:
+            failures.add(key, group["nodes"], "%s: %s" % (type(e).__name__, e))
+            continue
+        if action != "skipped":
+            done += 1
+            say("%s: %s (%s)" % (key, action, ", ".join(n["name"] for n in group["nodes"])))
+    return failures
+
+
 if __name__ == "__main__":
     sys.exit(0)

test_rgl_photo_backup.py

@@ -589,6 +589,14 @@ class Sidecars(TestCase):
         self.assertTrue(rb.iscomplete(self.dir, "111"))
         self.write("111.part", b"x")
         self.assertFalse(rb.iscomplete(self.dir, "111"))       # interrupted refetch
+
+    def test_sidenodes_survives_any_shape(self):
+        self.assertEqual(rb.sidenodes({}), [])
+        self.assertEqual(rb.sidenodes({"rgl": "nonsense"}), [])
+        self.assertEqual(rb.sidenodes({"rgl": {"nodes": "nonsense"}}), [])
+        self.assertEqual(rb.sidenodes({"rgl": {"nodes": ["nonsense"]}}), [])
+        self.assertEqual(rb.sidenodes(sidecar())[0]["name"], "A, E1")
+
     def test_listsidecars_unreadable_directory_is_an_error(self):
         with self.assertRaises(rb.BackupError):
             rb.listsidecars(os.path.join(self.dir, "nowhere"))
@@ -736,3 +744,191 @@ class Tags(TestCase):
     def test_photographer_whitespace_and_slashes(self):
         side = sidecar(copyright="  Some /  One ")
         self.assertIn("rgl/photographer/Some - One", rb.tagsfor(side))
+
+
+class FetchPhoto(TestCase):
+    def group(self, key="111", nodes=None):
+        nodes = nodes or [{"param": "A,_E1", "name": "A, E1",
+                           "url": "https://london.randomness.org.uk/wiki.cgi?A%2C_E1",
+                           "image": "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                           "page": "https://www.flickr.com/photos/x/111/",
+                           "copyright": "P", "licence": "https://creativecommons.org/licenses/by-sa/2.0/",
+                           "lat": "51.5", "long": "-0.1"}]
+        return {"key": key, "source": "flickr", "id": key, "image": nodes[0]["image"],
+                "page": nodes[0]["page"], "copyright": "P",
+                "licence": "https://creativecommons.org/licenses/by-sa/2.0/", "nodes": nodes}
+
+    def test_fetches_largest_and_writes_sidecar(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse(flickr_info("111")),
+            FakeResponse(flickr_sizes(("Small", 240, 180, "https://x/111_s.jpg"),
+                                      ("Original", 800, 600, "https://x/111_o.png"))),
+            FakeResponse(b"PNGDATA", headers={"Content-Type": "image/png"}))
+        side, action = rb.fetchphoto(self.dir, self.group(), "k")
+        self.assertEqual(action, "fetched")
+        self.assertEqual(side["file"], "111.png")
+        self.assertEqual(side["flickr"]["size_label"], "Original")
+        self.assertEqual(side["flickr"]["width"], 800)
+        self.assertEqual(side["rgl"]["nodes"][0]["param"], "A,_E1")
+        self.assertNotIn("image", side["rgl"]["nodes"][0])
+        self.assertTrue(rb.iscomplete(self.dir, "111"))
+        self.assertEqual(open(os.path.join(self.dir, "111.png"), "rb").read(), b"PNGDATA")
+        self.assertFalse(os.path.exists(os.path.join(self.dir, "111.part")))
+
+    def test_falls_back_when_largest_404s(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse(flickr_info("111")),
+            FakeResponse(flickr_sizes(("Large", 1024, 768, "https://x/111_b.jpg"),
+                                      ("Original", 800, 600, "https://x/111_o.jpg"))),
+            http_error(404),
+            FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"}))
+        side, action = rb.fetchphoto(self.dir, self.group(), "k")
+        self.assertEqual(side["flickr"]["size_label"], "Large")
+        self.assertIn("111_o.jpg", rb.OPEN.requests[2].full_url)
+
+    def test_gone_photo_raises_and_writes_nothing(self):
+        rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
+        with self.assertRaises(rb.PhotoGone):
+            rb.fetchphoto(self.dir, self.group(), "k")
+        self.assertEqual(os.listdir(self.dir), [])
+
+    def test_complete_photo_is_skipped_without_calls(self):
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        rb.OPEN = FakeOpen()
+        side, action = rb.fetchphoto(self.dir, self.group(), "k")
+        self.assertEqual(action, "skipped")
+
+    def test_changed_nodes_rewrite_sidecar_only(self):
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        rb.OPEN = FakeOpen()
+        g = self.group()
+        g["nodes"].append(dict(g["nodes"][0], param="B,_E2", name="B, E2", url="https://x/B"))
+        side, action = rb.fetchphoto(self.dir, g, "k")
+        self.assertEqual(action, "updated")
+        self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 2)
+        self.assertEqual(rb.STATS["rewritten"], 1)
+
+    def test_nodes_are_sorted_by_param(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse(flickr_info("111")),
+            FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))),
+            FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}))
+        g = self.group()
+        g["nodes"].insert(0, dict(g["nodes"][0], param="Z,_E9", name="Z, E9"))
+        side, action = rb.fetchphoto(self.dir, g, "k")
+        self.assertEqual([n["param"] for n in side["rgl"]["nodes"]], ["A,_E1", "Z,_E9"])
+        # and the same order the comparison uses, so a re-run rewrites nothing
+        self.assertEqual(rb.fetchphoto(self.dir, g, "k")[1], "skipped")
+
+    def test_downloadany_skips_an_empty_body(self):
+        rb.OPEN = FakeOpen(FakeResponse(b""),
+                           FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"}))
+        candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg",
+                       "media": "photo"},
+                      {"label": "Large", "width": 4, "height": 3, "source": "https://x/l.jpg",
+                       "media": "photo"}]
+        partial = os.path.join(self.dir, "111.part")
+        pick, ext = rb.downloadany(candidates, partial, "photo 111")
+        self.assertEqual((pick["label"], ext), ("Large", ".jpg"))
+
+    def test_downloadany_with_nothing_but_empty_bodies_fails(self):
+        rb.OPEN = FakeOpen(FakeResponse(b""))
+        candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg",
+                       "media": "photo"}]
+        with self.assertRaises(rb.BackupError) as cm:
+            rb.downloadany(candidates, os.path.join(self.dir, "111.part"), "photo 111")
+        self.assertIn("empty", str(cm.exception))
+
+    def test_partial_file_is_refetched(self):
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        self.write("111.part", b"half")
+        rb.OPEN = FakeOpen(FakeResponse(flickr_info("111")),
+                           FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))),
+                           FakeResponse(b"FULL", headers={"Content-Type": "image/jpeg"}))
+        side, action = rb.fetchphoto(self.dir, self.group(), "k")
+        self.assertEqual(action, "fetched")
+        self.assertEqual(open(os.path.join(self.dir, "111.jpg"), "rb").read(), b"FULL")
+
+    def test_sidecar_without_an_rgl_block_is_rewritten_not_fatal(self):
+        rb.writejson(rb.sidecarpath(self.dir, "111"), {"key": "111", "file": "111.jpg"})
+        self.write("111.jpg", b"x")
+        rb.OPEN = FakeOpen()
+        side, action = rb.fetchphoto(self.dir, self.group(), "k")
+        self.assertEqual(action, "updated")
+        self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 1)
+
+    def test_dry_run_calls_nothing(self):
+        rb.OPEN = FakeOpen()
+        side, action = rb.fetchphoto(self.dir, self.group(), "k", dryrun=True)
+        self.assertEqual(action, "would fetch")
+        self.assertEqual(os.listdir(self.dir), [])
+
+    def test_non_flickr_photo_downloads_image_url(self):
+        g = self.group(key="url-0123456789abcdef")
+        g["source"] = "url"
+        del g["id"]
+        g["image"] = g["nodes"][0]["image"] = "https://www.geograph.org.uk/p/1.jpg"
+        g["page"] = "https://www.geograph.org.uk/photo/1"
+        rb.OPEN = FakeOpen(FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"}))
+        side, action = rb.fetchphoto(self.dir, g, "k")
+        self.assertEqual(side["source"], "url")
+        self.assertEqual(side["page"], "https://www.geograph.org.uk/photo/1")
+        self.assertNotIn("flickr", side)
+        self.assertEqual(side["file"], "url-0123456789abcdef.jpg")
+
+
+class FetchPhase(TestCase):
+    def conf(self, **over):
+        conf = {"directory": self.dir, "index_url": "http://t/index", "limit": None,
+                "dry_run": False, "quiet": False}
+        conf.update(over)
+        return conf
+
+    def test_runs_over_index_and_collects_failures(self):
+        index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                             page="https://www.flickr.com/photos/x/111/"),
+                 index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg",
+                             page="https://www.flickr.com/photos/x/222/"),
+                 index_entry("C,_E3", "junk")]
+        rb.OPEN = FakeOpen(
+            FakeResponse(json.dumps(index).encode()),
+            FakeResponse(flickr_info("111")),
+            FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))),
+            FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}),
+            FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
+        failures = rb.fetchphase(self.conf(), "k")
+        self.assertTrue(rb.iscomplete(self.dir, "111"))
+        self.assertEqual(rb.STATS["fetched"], 1)
+        self.assertEqual(len(failures), 2)
+        reasons = [f[2] for f in failures.items]
+        self.assertTrue(any("Photo not found" in r for r in reasons), reasons)
+        self.assertTrue(any("not a url" in r for r in reasons), reasons)
+        names = [n for f in failures.items for n in f[1]]
+        self.assertIn("B, E2", names)
+
+    def test_limit_stops_early(self):
+        index = [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"),
+                 index_entry("B", "https://farm1.static.flickr.com/1/222_b_m.jpg")]
+        rb.OPEN = FakeOpen(
+            FakeResponse(json.dumps(index).encode()),
+            FakeResponse(flickr_info("111")),
+            FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))),
+            FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}))
+        rb.fetchphase(self.conf(limit=1), "k")
+        self.assertEqual(rb.STATS["fetched"], 1)
+        self.assertFalse(os.path.exists(rb.sidecarpath(self.dir, "222")))
+
+    def test_dry_run_creates_no_directory(self):
+        room = os.path.join(self.dir, "not-yet")
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps(
+            [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg")]).encode()))
+        rb.fetchphase(self.conf(directory=room, dry_run=True), "k")
+        self.assertFalse(os.path.exists(room))
+
+    def test_index_failure_is_fatal(self):
+        rb.OPEN = FakeOpen(http_error(404))
+        with self.assertRaises(rb.BackupError):
+            rb.fetchphase(self.conf(), "k")