bob/rgl-photo-backup
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fetch every wiki photo at its largest size into the backup directory

Browse Source 
Walk the grouped index, ask flickr for each new photo, download the best
size that actually serves, and write the sidecar only once the file is
renamed into place. Photos already complete cost no flickr calls; a change
in which pages use one rewrites just the sidecar. Every failure is
recorded with the pages affected and the run carries on.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
bobandClaude Fable 5.1 committed 2026-09-10 12:07:45 +01:00
1 parent 2774267425
commit 5549217db5
2 files changed
+336
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test_rgl_photo_backup.py
+196
View File
@@ -589,6 +589,14 @@ class Sidecars(TestCase):
self.assertTrue(rb.iscomplete(self.dir, "111"))
self.write("111.part", b"x")
self.assertFalse(rb.iscomplete(self.dir, "111")) # interrupted refetch
def test_sidenodes_survives_any_shape(self):
self.assertEqual(rb.sidenodes({}), [])
self.assertEqual(rb.sidenodes({"rgl": "nonsense"}), [])
self.assertEqual(rb.sidenodes({"rgl": {"nodes": "nonsense"}}), [])
self.assertEqual(rb.sidenodes({"rgl": {"nodes": ["nonsense"]}}), [])
self.assertEqual(rb.sidenodes(sidecar())[0]["name"], "A, E1")
def test_listsidecars_unreadable_directory_is_an_error(self):
with self.assertRaises(rb.BackupError):
rb.listsidecars(os.path.join(self.dir, "nowhere"))
@@ -736,3 +744,191 @@ class Tags(TestCase):
def test_photographer_whitespace_and_slashes(self):
side = sidecar(copyright=" Some / One ")
self.assertIn("rgl/photographer/Some - One", rb.tagsfor(side))
class FetchPhoto(TestCase):
def group(self, key="111", nodes=None):
nodes = nodes or [{"param": "A,_E1", "name": "A, E1",
"url": "https://london.randomness.org.uk/wiki.cgi?A%2C_E1",
"image": "https://farm1.static.flickr.com/1/111_a_m.jpg",
"page": "https://www.flickr.com/photos/x/111/",
"copyright": "P", "licence": "https://creativecommons.org/licenses/by-sa/2.0/",
"lat": "51.5", "long": "-0.1"}]
return {"key": key, "source": "flickr", "id": key, "image": nodes[0]["image"],
"page": nodes[0]["page"], "copyright": "P",
"licence": "https://creativecommons.org/licenses/by-sa/2.0/", "nodes": nodes}
def test_fetches_largest_and_writes_sidecar(self):
rb.OPEN = FakeOpen(
FakeResponse(flickr_info("111")),
FakeResponse(flickr_sizes(("Small", 240, 180, "https://x/111_s.jpg"),
("Original", 800, 600, "https://x/111_o.png"))),
FakeResponse(b"PNGDATA", headers={"Content-Type": "image/png"}))
side, action = rb.fetchphoto(self.dir, self.group(), "k")
self.assertEqual(action, "fetched")
self.assertEqual(side["file"], "111.png")
self.assertEqual(side["flickr"]["size_label"], "Original")
self.assertEqual(side["flickr"]["width"], 800)
self.assertEqual(side["rgl"]["nodes"][0]["param"], "A,_E1")
self.assertNotIn("image", side["rgl"]["nodes"][0])
self.assertTrue(rb.iscomplete(self.dir, "111"))
self.assertEqual(open(os.path.join(self.dir, "111.png"), "rb").read(), b"PNGDATA")
self.assertFalse(os.path.exists(os.path.join(self.dir, "111.part")))
def test_falls_back_when_largest_404s(self):
rb.OPEN = FakeOpen(
FakeResponse(flickr_info("111")),
FakeResponse(flickr_sizes(("Large", 1024, 768, "https://x/111_b.jpg"),
("Original", 800, 600, "https://x/111_o.jpg"))),
http_error(404),
FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"}))
side, action = rb.fetchphoto(self.dir, self.group(), "k")
self.assertEqual(side["flickr"]["size_label"], "Large")
self.assertIn("111_o.jpg", rb.OPEN.requests[2].full_url)
def test_gone_photo_raises_and_writes_nothing(self):
rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
with self.assertRaises(rb.PhotoGone):
rb.fetchphoto(self.dir, self.group(), "k")
self.assertEqual(os.listdir(self.dir), [])
def test_complete_photo_is_skipped_without_calls(self):
rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
self.write("111.jpg", b"x")
rb.OPEN = FakeOpen()
side, action = rb.fetchphoto(self.dir, self.group(), "k")
self.assertEqual(action, "skipped")
def test_changed_nodes_rewrite_sidecar_only(self):
rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
self.write("111.jpg", b"x")
rb.OPEN = FakeOpen()
g = self.group()
g["nodes"].append(dict(g["nodes"][0], param="B,_E2", name="B, E2", url="https://x/B"))
side, action = rb.fetchphoto(self.dir, g, "k")
self.assertEqual(action, "updated")
self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 2)
self.assertEqual(rb.STATS["rewritten"], 1)
def test_nodes_are_sorted_by_param(self):
rb.OPEN = FakeOpen(
FakeResponse(flickr_info("111")),
FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))),
FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}))
g = self.group()
g["nodes"].insert(0, dict(g["nodes"][0], param="Z,_E9", name="Z, E9"))
side, action = rb.fetchphoto(self.dir, g, "k")
self.assertEqual([n["param"] for n in side["rgl"]["nodes"]], ["A,_E1", "Z,_E9"])
# and the same order the comparison uses, so a re-run rewrites nothing
self.assertEqual(rb.fetchphoto(self.dir, g, "k")[1], "skipped")
def test_downloadany_skips_an_empty_body(self):
rb.OPEN = FakeOpen(FakeResponse(b""),
FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"}))
candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg",
"media": "photo"},
{"label": "Large", "width": 4, "height": 3, "source": "https://x/l.jpg",
"media": "photo"}]
partial = os.path.join(self.dir, "111.part")
pick, ext = rb.downloadany(candidates, partial, "photo 111")
self.assertEqual((pick["label"], ext), ("Large", ".jpg"))
def test_downloadany_with_nothing_but_empty_bodies_fails(self):
rb.OPEN = FakeOpen(FakeResponse(b""))
candidates = [{"label": "Original", "width": 8, "height": 6, "source": "https://x/o.jpg",
"media": "photo"}]
with self.assertRaises(rb.BackupError) as cm:
rb.downloadany(candidates, os.path.join(self.dir, "111.part"), "photo 111")
self.assertIn("empty", str(cm.exception))
def test_partial_file_is_refetched(self):
rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
self.write("111.jpg", b"x")
self.write("111.part", b"half")
rb.OPEN = FakeOpen(FakeResponse(flickr_info("111")),
FakeResponse(flickr_sizes(("Original", 800, 600, "https://x/111_o.jpg"))),
FakeResponse(b"FULL", headers={"Content-Type": "image/jpeg"}))
side, action = rb.fetchphoto(self.dir, self.group(), "k")
self.assertEqual(action, "fetched")
self.assertEqual(open(os.path.join(self.dir, "111.jpg"), "rb").read(), b"FULL")
def test_sidecar_without_an_rgl_block_is_rewritten_not_fatal(self):
rb.writejson(rb.sidecarpath(self.dir, "111"), {"key": "111", "file": "111.jpg"})
self.write("111.jpg", b"x")
rb.OPEN = FakeOpen()
side, action = rb.fetchphoto(self.dir, self.group(), "k")
self.assertEqual(action, "updated")
self.assertEqual(len(rb.readsidecar(rb.sidecarpath(self.dir, "111"))["rgl"]["nodes"]), 1)
def test_dry_run_calls_nothing(self):
rb.OPEN = FakeOpen()
side, action = rb.fetchphoto(self.dir, self.group(), "k", dryrun=True)
self.assertEqual(action, "would fetch")
self.assertEqual(os.listdir(self.dir), [])
def test_non_flickr_photo_downloads_image_url(self):
g = self.group(key="url-0123456789abcdef")
g["source"] = "url"
del g["id"]
g["image"] = g["nodes"][0]["image"] = "https://www.geograph.org.uk/p/1.jpg"
g["page"] = "https://www.geograph.org.uk/photo/1"
rb.OPEN = FakeOpen(FakeResponse(b"JPG", headers={"Content-Type": "image/jpeg"}))
side, action = rb.fetchphoto(self.dir, g, "k")
self.assertEqual(side["source"], "url")
self.assertEqual(side["page"], "https://www.geograph.org.uk/photo/1")
self.assertNotIn("flickr", side)
self.assertEqual(side["file"], "url-0123456789abcdef.jpg")
class FetchPhase(TestCase):
def conf(self, **over):
conf = {"directory": self.dir, "index_url": "http://t/index", "limit": None,
"dry_run": False, "quiet": False}
conf.update(over)
return conf
def test_runs_over_index_and_collects_failures(self):
index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
page="https://www.flickr.com/photos/x/111/"),
index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg",
page="https://www.flickr.com/photos/x/222/"),
index_entry("C,_E3", "junk")]
rb.OPEN = FakeOpen(
FakeResponse(json.dumps(index).encode()),
FakeResponse(flickr_info("111")),
FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))),
FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}),
FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
failures = rb.fetchphase(self.conf(), "k")
self.assertTrue(rb.iscomplete(self.dir, "111"))
self.assertEqual(rb.STATS["fetched"], 1)
self.assertEqual(len(failures), 2)
reasons = [f[2] for f in failures.items]
self.assertTrue(any("Photo not found" in r for r in reasons), reasons)
self.assertTrue(any("not a url" in r for r in reasons), reasons)
names = [n for f in failures.items for n in f[1]]
self.assertIn("B, E2", names)
def test_limit_stops_early(self):
index = [index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"),
index_entry("B", "https://farm1.static.flickr.com/1/222_b_m.jpg")]
rb.OPEN = FakeOpen(
FakeResponse(json.dumps(index).encode()),
FakeResponse(flickr_info("111")),
FakeResponse(flickr_sizes(("Original", 8, 6, "https://x/111_o.jpg"))),
FakeResponse(b"J", headers={"Content-Type": "image/jpeg"}))
rb.fetchphase(self.conf(limit=1), "k")
self.assertEqual(rb.STATS["fetched"], 1)
self.assertFalse(os.path.exists(rb.sidecarpath(self.dir, "222")))
def test_dry_run_creates_no_directory(self):
room = os.path.join(self.dir, "not-yet")
rb.OPEN = FakeOpen(FakeResponse(json.dumps(
[index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg")]).encode()))
rb.fetchphase(self.conf(directory=room, dry_run=True), "k")
self.assertFalse(os.path.exists(room))
def test_index_failure_is_fatal(self):
rb.OPEN = FakeOpen(http_error(404))
with self.assertRaises(rb.BackupError):
rb.fetchphase(self.conf(), "k")