Remember why photos failed and report the pages without one
Each run now writes .failures.json in the backup directory: the last reason every photo failed and the wiki pages that use it, dropping any photo that has since arrived. A new --report mode reads the wiki index and that file and lists every page whose photo is not on disk, with the reason or "not yet attempted", calling neither flickr nor immich. That is what tells a wiki editor which photos have gone from flickr and need replacing. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
3 files changed
+195
-2
No files matched your search
+114
-1
@@ -342,6 +342,11 @@ class Settings(TestCase):
|
||||
rb.settings(args, environ={rb.ENV + "ATTEMPTS": "lots"})
|
||||
self.assertIn("ATTEMPTS", str(cm.exception))
|
||||
|
||||
def test_report_flag(self):
|
||||
base = ["--settings", os.path.join(self.dir, "none"), "/d"]
|
||||
self.assertFalse(rb.settings(rb.parseargs(base), environ={})["report"])
|
||||
self.assertTrue(rb.settings(rb.parseargs(base + ["--report"]), environ={})["report"])
|
||||
|
||||
def test_phase_flags(self):
|
||||
base = ["--settings", os.path.join(self.dir, "none"), "/d"]
|
||||
self.assertFalse(rb.settings(rb.parseargs(base + ["--sync-only"]), environ={})["fetch"])
|
||||
@@ -653,6 +658,7 @@ class Sidecars(TestCase):
|
||||
rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg"))
|
||||
rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
|
||||
rb.writejson(os.path.join(self.dir, rb.HASHES), {})
|
||||
rb.writejson(os.path.join(self.dir, rb.FAILURES), {})
|
||||
self.assertEqual([k for k, _ in rb.listsidecars(self.dir)], ["111", "222"])
|
||||
|
||||
def test_utcnow_shape(self):
|
||||
@@ -1408,7 +1414,7 @@ class Run(TestCase):
|
||||
conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"),
|
||||
"immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"),
|
||||
"index_url": "http://t/index", "fetch": True, "sync": True,
|
||||
"dry_run": False, "limit": None, "pace": 0, "quiet": False,
|
||||
"dry_run": False, "report": False, "limit": None, "pace": 0, "quiet": False,
|
||||
"attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False}
|
||||
conf.update(over)
|
||||
return conf
|
||||
@@ -1490,3 +1496,110 @@ class Run(TestCase):
|
||||
def test_unreadable_settings_file_gives_exit_2(self):
|
||||
self.assertEqual(rb.main(["--settings", self.dir, self.dir]), 2)
|
||||
self.assertIn(self.dir, self.stderr.getvalue())
|
||||
|
||||
|
||||
class FailureRecords(TestCase):
|
||||
def failures(self, *items):
|
||||
f = rb.Failures()
|
||||
for key, names, reason in items:
|
||||
f.add(key, [{"name": n} for n in names], reason)
|
||||
return f
|
||||
|
||||
def test_missing_file_is_empty(self):
|
||||
self.assertEqual(rb.loadfailures(self.dir), {})
|
||||
|
||||
def test_records_merge_and_drop_completed(self):
|
||||
rb.recordfailures(self.dir, self.failures(("111", ["A, E1"], "gone"),
|
||||
("222", ["B, E2"], "HTTP 404")))
|
||||
held = rb.loadfailures(self.dir)
|
||||
self.assertEqual(sorted(held), ["111", "222"])
|
||||
self.assertEqual(held["111"]["reason"], "gone")
|
||||
self.assertEqual(held["111"]["pages"], ["A, E1"])
|
||||
self.assertRegex(held["111"]["when"], r"^\d{4}-")
|
||||
# 111 later succeeds; 333 newly fails; 222 untouched this run but still missing
|
||||
rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
|
||||
self.write("111.jpg", b"x")
|
||||
rb.recordfailures(self.dir, self.failures(("333", ["C, E3"], "timeout")))
|
||||
held = rb.loadfailures(self.dir)
|
||||
self.assertEqual(sorted(held), ["222", "333"])
|
||||
|
||||
def test_unreadable_file_is_treated_as_empty(self):
|
||||
self.write(rb.FAILURES, "{not json")
|
||||
self.assertEqual(rb.loadfailures(self.dir), {})
|
||||
|
||||
|
||||
class Report(TestCase):
|
||||
def conf(self, **over):
|
||||
conf = {"directory": self.dir, "index_url": "http://t/index", "report": True,
|
||||
"fetch": True, "sync": True, "dry_run": False, "limit": None, "quiet": False,
|
||||
"flickr_auth": os.path.join(self.dir, "missing.yml"),
|
||||
"immich_auth": os.path.join(self.dir, "missing.yml"),
|
||||
"pace": 0, "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua",
|
||||
"debug": False}
|
||||
conf.update(over)
|
||||
return conf
|
||||
|
||||
def index(self):
|
||||
return json.dumps([
|
||||
index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
|
||||
page="https://www.flickr.com/photos/x/111/"),
|
||||
index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg",
|
||||
page="https://www.flickr.com/photos/x/222/"),
|
||||
index_entry("C,_E3", "https://farm1.static.flickr.com/1/333_c_m.jpg"),
|
||||
index_entry("D,_E4", "junk"),
|
||||
]).encode()
|
||||
|
||||
def test_lists_missing_photos_with_pages_and_reasons(self):
|
||||
rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
|
||||
self.write("111.jpg", b"x")
|
||||
rb.writejson(os.path.join(self.dir, rb.FAILURES),
|
||||
{"222": {"reason": "photo 222: HTTP 404 Not Found", "pages": ["B, E2"],
|
||||
"when": "2026-09-13T10:00:00Z"}})
|
||||
rb.OPEN = FakeOpen(FakeResponse(self.index()))
|
||||
self.assertEqual(rb.run(self.conf()), 0)
|
||||
out = self.stdout.getvalue()
|
||||
self.assertNotIn("111", out)
|
||||
self.assertIn("222: photo 222: HTTP 404 Not Found (2026-09-13)", out)
|
||||
self.assertIn(" B, E2 https://london.randomness.org.uk/wiki.cgi?B%2C_E2", out)
|
||||
self.assertIn("333: not yet attempted", out)
|
||||
self.assertIn(" C, E3 ", out)
|
||||
self.assertIn("D,_E4: node_image is not a url", out)
|
||||
self.assertIn("2 of 3 photos missing, 1 node with an unusable image", out)
|
||||
self.assertEqual(len(rb.OPEN.requests), 1) # the index only
|
||||
|
||||
def test_report_prints_even_when_quiet(self):
|
||||
rb.OPEN = FakeOpen(FakeResponse(self.index()))
|
||||
rb.run(self.conf(quiet=True))
|
||||
self.assertIn("3 of 3 photos missing", self.stdout.getvalue())
|
||||
|
||||
def test_report_needs_no_credentials(self):
|
||||
rb.OPEN = FakeOpen(FakeResponse(b"[]"))
|
||||
self.assertEqual(rb.run(self.conf()), 0)
|
||||
self.assertIn("0 of 0 photos missing", self.stdout.getvalue())
|
||||
|
||||
|
||||
class RunRecordsFailures(TestCase):
|
||||
def conf(self, **over):
|
||||
conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"),
|
||||
"immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"),
|
||||
"index_url": "http://t/index", "fetch": True, "sync": False, "report": False,
|
||||
"dry_run": False, "limit": None, "pace": 0, "quiet": False,
|
||||
"attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False}
|
||||
conf.update(over)
|
||||
return conf
|
||||
|
||||
def test_failed_run_writes_the_failures_file(self):
|
||||
index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")]
|
||||
rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()),
|
||||
FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
|
||||
self.assertEqual(rb.run(self.conf()), 1)
|
||||
held = rb.loadfailures(self.dir)
|
||||
self.assertIn("111", held)
|
||||
self.assertIn("Photo not found", held["111"]["reason"])
|
||||
self.assertEqual(held["111"]["pages"], ["A, E1"])
|
||||
|
||||
def test_dry_run_writes_nothing(self):
|
||||
index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")]
|
||||
rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()))
|
||||
rb.run(self.conf(dry_run=True))
|
||||
self.assertFalse(os.path.exists(os.path.join(self.dir, rb.FAILURES)))
|
||||
Reference in new issue
Block a user