diff --git a/rgl-photo-backup b/rgl-photo-backup index 9ae6965..c95c620 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -779,5 +779,145 @@ def tagsfor(side): return tags +class Failures: + def __init__(self): + self.items = [] + + def add(self, key, nodes, reason): + names = [n.get("name", n.get("param", "")) for n in nodes] + self.items.append((key, names, str(reason))) + STATS["failed"] += 1 + warn("%s (%s): %s" % (key, ", ".join(names) or "no pages", reason)) + + def __len__(self): + return len(self.items) + + +def nodeentry(node): + return {k: node.get(k, "") for k in ("param", "name", "url", "lat", "long")} + +# By node name, so a sidecar and the description built from it do not change +# just because the index listed the same pages in another order. +def nodeentries(nodes): + return [nodeentry(n) for n in sorted(nodes, key=lambda n: n.get("param", ""))] + +# The pages a sidecar records, for naming a failure. A sidecar on disk can be +# any shape at all, and the failure path must not itself raise. +def sidenodes(side): + rgl = side.get("rgl") + nodes = rgl.get("nodes") if isinstance(rgl, dict) else None + return [n for n in nodes if isinstance(n, dict)] if isinstance(nodes, list) else [] + +def buildsidecar(group, info, pick, filename): + side = { + "key": group["key"], + "file": filename, + "source": group["source"], + "rgl": {"copyright": group["copyright"], "licence": group["licence"], + "nodes": nodeentries(group["nodes"])}, + "fetched": utcnow(), + } + if group["source"] == "flickr": + side["flickr"] = dict(info, size_label=pick["label"], + width=pick["width"], height=pick["height"]) + else: + side["page"] = group["page"] or group["image"] + return side + +# +# Try each candidate url in turn, best first; a 404 on one size means try the +# next, anything else is a real failure. Returns (candidate, extension). +# +def downloadany(candidates, partial, context): + last = None + for pick in candidates: + try: + ctype, written = download(pick["source"], context, partial, pace=True) + except HTTPFailure as e: + if e.code != 404: + raise + last = e + continue + if written == 0: + last = BackupError("%s: empty response" % context) + continue + return (pick, extensionfor(pick["source"], ctype)) + raise last or BackupError("%s: no downloadable size" % context) + +def fetchphoto(directory, group, flickrkey, dryrun=False): + key = group["key"] + context = "photo %s" % key + sidepath = sidecarpath(directory, key) + + if iscomplete(directory, key): + held = readsidecar(sidepath) + if not isinstance(held.get("rgl"), dict): + held["rgl"] = {} + wanted = nodeentries(group["nodes"]) + if held["rgl"].get("nodes") == wanted and held["rgl"].get("copyright") == group["copyright"] \ + and held["rgl"].get("licence") == group["licence"]: + STATS["skipped"] += 1 + return (held, "skipped") + held["rgl"].update(nodes=wanted, copyright=group["copyright"], licence=group["licence"]) + if not dryrun: + writejson(sidepath, held) + STATS["rewritten"] += 1 + return (held, "updated") + + if dryrun: + return (None, "would fetch") + + if group["source"] == "flickr": + info = getinfo(group["id"], flickrkey) + candidates = picksizes(getsizes(group["id"], flickrkey)) + if not candidates: + raise BackupError("%s: flickr lists no downloadable sizes" % context) + else: + info = None + candidates = [{"label": "", "width": 0, "height": 0, "source": group["image"], "media": "photo"}] + + partial = os.path.join(directory, key + ".part") + try: + pick, ext = downloadany(candidates, partial, context) + filename = key + ext + side = buildsidecar(group, info, pick, filename) + os.replace(partial, os.path.join(directory, filename)) + writejson(sidepath, side) + finally: + if os.path.exists(partial): + os.remove(partial) + STATS["fetched"] += 1 + return (side, "fetched") + +def fetchphase(conf, flickrkey): + failures = Failures() + if not conf["dry_run"]: + os.makedirs(conf["directory"], exist_ok=True) + nodes = loadindex(conf["index_url"]) + groups, rejected = groupphotos(nodes) + say("index: %d nodes, %d photos on %d of them" + % (len(nodes), len(groups), sum(len(g["nodes"]) for g in groups.values()))) + for node, reason in rejected: + failures.add(node["param"], [node], reason) + + done = 0 + for key in sorted(groups): + if conf["limit"] is not None and done >= conf["limit"]: + break + group = groups[key] + try: + side, action = fetchphoto(conf["directory"], group, flickrkey, conf["dry_run"]) + except BackupError as e: + failures.add(key, group["nodes"], e) + continue + except OSError as e: + failures.add(key, group["nodes"], "%s: %s" % (type(e).__name__, e)) + continue + if action != "skipped": + done += 1 + say("%s: %s (%s)" % (key, action, ", ".join(n["name"] for n in group["nodes"]))) + return failures + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index 177e311..70d13fc 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -589,6 +589,14 @@ class Sidecars(TestCase): self.assertTrue(rb.iscomplete(self.dir, "111")) self.write("111.part", b"x") self.assertFalse(rb.iscomplete(self.dir, "111")) # interrupted refetch + + def test_sidenodes_survives_any_shape(self): + self.assertEqual(rb.sidenodes({}), []) + self.assertEqual(rb.sidenodes({"rgl": "nonsense"}), []) + self.assertEqual(rb.sidenodes({"rgl": {"nodes": "nonsense"}}), []) + self.assertEqual(rb.sidenodes({"rgl": {"nodes": ["nonsense"]}}), []) + self.assertEqual(rb.sidenodes(sidecar())[0]["name"], "A, E1") + def test_listsidecars_unreadable_directory_is_an_error(self): with self.assertRaises(rb.BackupError): rb.listsidecars(os.path.join(self.dir, "nowhere")) @@ -736,3 +744,191 @@ class Tags(TestCase): def test_photographer_whitespace_and_slashes(self): side = sidecar(copyright=" Some / One ") self.assertIn("rgl/photographer/Some - One", rb.tagsfor(side)) + + +class FetchPhoto(TestCase): + def group(self, key="111", nodes=None): + nodes = nodes or [{"param": "A,_E1", "name": "A, E1", + "url": "https://london.randomness.org.uk/wiki.cgi?A%2C_E1", + "image": "https://farm1.static.flickr.com/1/111_a_m.jpg", + "page": "https://www.flickr.com/photos/x/111/", + "copyright": "P", "licence": "https://creativecommons.org/licenses/by-sa/2.0/", + "lat": "51.5", "long": "-0.1"}] + return {"key": key, "source": "flickr", "id": key, "image": nodes[0]["image"], + "page": nodes[0]["page"], "copyright": "P", + "licence": "https://creativecommons.org/licenses/by-sa/2.0/", "nodes": nodes} + + def test_fetches_largest_and_writes_sidecar(self): + rb.OPEN = FakeOpen( + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Small", 240, 180, "https://x/111_s.jpg"), + ("Original", 800, 600, "https://x/111_o.png"))), + FakeResponse(b"PNGDATA", headers={"Content-Type": "image/png"})) + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "fetched") + self.assertEqual(side["file"], "111.png") + self.assertEqual(side["flickr"]["size_label"], "Original") + self.assertEqual(side["flickr"]["width"], 800) + self.assertEqual(side["rgl"]["nodes"][0]["param"], "A,_E1") + self.assertNotIn("image", side["rgl"]["nodes"][0]) + self.assertTrue(rb.iscomplete(self.dir, "111")) + self.assertEqual(open(os.path.join(self.dir, "111.png"), "rb").read(), b"PNGDATA") + self.assertFalse(os.path.exists(os.path.join(self.dir, "111.part"))) + + def test_falls_back_when_largest_404s(self): + rb.OPEN = FakeOpen( + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Large", 1024, 768, "https://x/111_b.jpg"), + ("Original", 800, 600, "https://x/111_o.jpg"))), + http_error(404), + FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"})) + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(side["flickr"]["size_label"], "Large") + self.assertIn("111_o.jpg", rb.OPEN.requests[2].full_url) + + def test_gone_photo_raises_and_writes_nothing(self): + rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"})) + with self.assertRaises(rb.PhotoGone): + rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(os.listdir(self.dir), []) + + def test_complete_photo_is_skipped_without_calls(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + rb.OPEN = FakeOpen() + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "skipped") + + def test_changed_nodes_rewrite_sidecar_only(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + rb.OPEN = FakeOpen() + g = self.group() + g["nodes"].append(dict(g["nodes"][0], param="B,_E2", name="B, E2", url="https://x/B")) + side, action = rb.fetchphoto(self.dir, g, "k") + self.assertEqual(action, "updated") + self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 2) + self.assertEqual(rb.STATS["rewritten"], 1) + + def test_nodes_are_sorted_by_param(self): + rb.OPEN = FakeOpen( + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))), + FakeResponse(b"J", headers={"Content-Type": "image/jpeg"})) + g = self.group() + g["nodes"].insert(0, dict(g["nodes"][0], param="Z,_E9", name="Z, E9")) + side, action = rb.fetchphoto(self.dir, g, "k") + self.assertEqual([n["param"] for n in side["rgl"]["nodes"]], ["A,_E1", "Z,_E9"]) + # and the same order the comparison uses, so a re-run rewrites nothing + self.assertEqual(rb.fetchphoto(self.dir, g, "k")[1], "skipped") + + def test_downloadany_skips_an_empty_body(self): + rb.OPEN = FakeOpen(FakeResponse(b""), + FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"})) + candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg", + "media": "photo"}, + {"label": "Large", "width": 4, "height": 3, "source": "https://x/l.jpg", + "media": "photo"}] + partial = os.path.join(self.dir, "111.part") + pick, ext = rb.downloadany(candidates, partial, "photo 111") + self.assertEqual((pick["label"], ext), ("Large", ".jpg")) + + def test_downloadany_with_nothing_but_empty_bodies_fails(self): + rb.OPEN = FakeOpen(FakeResponse(b"")) + candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg", + "media": "photo"}] + with self.assertRaises(rb.BackupError) as cm: + rb.downloadany(candidates, os.path.join(self.dir, "111.part"), "photo 111") + self.assertIn("empty", str(cm.exception)) + + def test_partial_file_is_refetched(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + self.write("111.part", b"half") + rb.OPEN = FakeOpen(FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))), + FakeResponse(b"FULL", headers={"Content-Type": "image/jpeg"})) + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "fetched") + self.assertEqual(open(os.path.join(self.dir, "111.jpg"), "rb").read(), b"FULL") + + def test_sidecar_without_an_rgl_block_is_rewritten_not_fatal(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), {"key": "111", "file": "111.jpg"}) + self.write("111.jpg", b"x") + rb.OPEN = FakeOpen() + side, action = rb.fetchphoto(self.dir, self.group(), "k") + self.assertEqual(action, "updated") + self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 1) + + def test_dry_run_calls_nothing(self): + rb.OPEN = FakeOpen() + side, action = rb.fetchphoto(self.dir, self.group(), "k", dryrun=True) + self.assertEqual(action, "would fetch") + self.assertEqual(os.listdir(self.dir), []) + + def test_non_flickr_photo_downloads_image_url(self): + g = self.group(key="url-0123456789abcdef") + g["source"] = "url" + del g["id"] + g["image"] = g["nodes"][0]["image"] = "https://www.geograph.org.uk/p/1.jpg" + g["page"] = "https://www.geograph.org.uk/photo/1" + rb.OPEN = FakeOpen(FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"})) + side, action = rb.fetchphoto(self.dir, g, "k") + self.assertEqual(side["source"], "url") + self.assertEqual(side["page"], "https://www.geograph.org.uk/photo/1") + self.assertNotIn("flickr", side) + self.assertEqual(side["file"], "url-0123456789abcdef.jpg") + + +class FetchPhase(TestCase): + def conf(self, **over): + conf = {"directory": self.dir, "index_url": "http://t/index", "limit": None, + "dry_run": False, "quiet": False} + conf.update(over) + return conf + + def test_runs_over_index_and_collects_failures(self): + index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg", + page="https://www.flickr.com/photos/x/111/"), + index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg", + page="https://www.flickr.com/photos/x/222/"), + index_entry("C,_E3", "junk")] + rb.OPEN = FakeOpen( + FakeResponse(json.dumps(index).encode()), + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))), + FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}), + FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"})) + failures = rb.fetchphase(self.conf(), "k") + self.assertTrue(rb.iscomplete(self.dir, "111")) + self.assertEqual(rb.STATS["fetched"], 1) + self.assertEqual(len(failures), 2) + reasons = [f[2] for f in failures.items] + self.assertTrue(any("Photo not found" in r for r in reasons), reasons) + self.assertTrue(any("not a url" in r for r in reasons), reasons) + names = [n for f in failures.items for n in f[1]] + self.assertIn("B, E2", names) + + def test_limit_stops_early(self): + index = [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"), + index_entry("B", "https://farm1.static.flickr.com/1/222_b_m.jpg")] + rb.OPEN = FakeOpen( + FakeResponse(json.dumps(index).encode()), + FakeResponse(flickr_info("111")), + FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))), + FakeResponse(b"J", headers={"Content-Type": "image/jpeg"})) + rb.fetchphase(self.conf(limit=1), "k") + self.assertEqual(rb.STATS["fetched"], 1) + self.assertFalse(os.path.exists(rb.sidecarpath(self.dir, "222"))) + + def test_dry_run_creates_no_directory(self): + room = os.path.join(self.dir, "not-yet") + rb.OPEN = FakeOpen(FakeResponse(json.dumps( + [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg")]).encode())) + rb.fetchphase(self.conf(directory=room, dry_run=True), "k") + self.assertFalse(os.path.exists(room)) + + def test_index_failure_is_fatal(self): + rb.OPEN = FakeOpen(http_error(404)) + with self.assertRaises(rb.BackupError): + rb.fetchphase(self.conf(), "k")