Remember why photos failed and report the pages without one

Each run now writes .failures.json in the backup directory: the last
reason every photo failed and the wiki pages that use it, dropping any
photo that has since arrived. A new --report mode reads the wiki index
and that file and lists every page whose photo is not on disk, with the
reason or "not yet attempted", calling neither flickr nor immich. That is
what tells a wiki editor which photos have gone from flickr and need
replacing.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

README.md

@@ -42,6 +42,7 @@ Make the credential files `chmod 600`.
 ./rgl-photo-backup --settings other.yml DIR   # a different settings file
 ./rgl-photo-backup --debug DIR                # log every request to stderr
 ./rgl-photo-backup -q DIR                     # same as --quiet
+./rgl-photo-backup --report DIR               # which pages have no photo on disk, and why
 ```
 
 The first run downloads about 4800 photos, one Flickr info call and one
@@ -117,6 +118,13 @@ image URL (`url-0f4b1c2d3e4f5a6b.jpg`), with a matching sidecar. The
 directory is a complete backup on its own. A `.part` file is an interrupted
 download and is refetched next run.
 
+`.failures.json` records the last reason each photo failed, with the pages
+that use it. A photo that later arrives drops out of it. `--report` reads the
+wiki index and that file, and lists every page whose photo is not on disk
+with the reason, or `not yet attempted`, without calling Flickr or Immich, so
+it is the way to find photos that have gone from Flickr and need replacing
+on the wiki.
+
 ## Running under cron
 
 ```cron
@@ -141,6 +149,7 @@ A flag beats the environment, which beats the file, which beats the default.
 | `fetch` | `--sync-only` sets false | | `true` |
 | `sync` | `--fetch-only` sets false | | `true` |
 | `dry_run` | `--dry-run` | | `false` |
+| `report` | `--report` | | `false` |
 | `limit` | `--limit N` | | unlimited |
 | `pace` | `--pace SECONDS` | `RGL_PHOTO_BACKUP_PACE` | `1` |
 | `quiet` | `--quiet`, `-q` | | `false` |

rgl-photo-backup

@@ -38,6 +38,7 @@ ALBUM          = "RGL"
 TAG_ROOT       = "rgl"
 DEVICE_ID      = "rgl-photo-backup"
 HASHES         = ".immich-hashes.json"
+FAILURES       = ".failures.json"
 
 PAGE           = 1000            # immich search page
 BATCH          = 500             # ids per album/tag write
@@ -59,6 +60,7 @@ DEFAULTS = {
     "fetch":       True,
     "sync":        True,
     "dry_run":     False,
+    "report":      False,
     "limit":       None,
     "pace":        1.0,
     "quiet":       False,
@@ -158,6 +160,7 @@ SETTING_KINDS = {
     "fetch":       bool,
     "sync":        bool,
     "dry_run":     bool,
+    "report":      bool,
     "limit":       int,
     "pace":        float,
     "quiet":       bool,
@@ -209,6 +212,9 @@ def parseargs(argv=None):
                         help="disk to immich, no RGL or flickr calls")
     parser.add_argument("--dry-run", action="store_true", default=None,
                         help="report what would be fetched and written, change nothing")
+    parser.add_argument("--report", action="store_true", default=None,
+                        help="list the wiki photos not on disk, with their pages and "
+                             "the last recorded reason; reads the index only")
     parser.add_argument("--limit", type=int, metavar="N",
                         help="stop after N photos, for a trial run")
     parser.add_argument("--pace", type=float, metavar="SECONDS",
@@ -257,6 +263,7 @@ def settings(args, environ=None):
         "fetch":       pick("fetch", False if args.sync_only else None, DEFAULTS["fetch"]),
         "sync":        pick("sync", False if args.fetch_only else None, DEFAULTS["sync"]),
         "dry_run":     pick("dry_run", args.dry_run, DEFAULTS["dry_run"]),
+        "report":      pick("report", args.report, DEFAULTS["report"]),
         "limit":       pick("limit", args.limit, DEFAULTS["limit"]),
         "pace":        pick("pace", args.pace, DEFAULTS["pace"]),
         "quiet":       pick("quiet", args.quiet, DEFAULTS["quiet"]),
@@ -665,7 +672,7 @@ def listsidecars(directory):
     except OSError as e:
         raise BackupError("%s: cannot be listed: %s" % (directory, e.strerror or e))
     for name in names:
-        if not name.endswith(".json") or name == HASHES:
+        if not name.endswith(".json") or name in (HASHES, FAILURES):
             continue
         side = readsidecar(os.path.join(directory, name))
         if side:
@@ -811,6 +818,28 @@ class Failures:
         return len(self.items)
 
 
+#
+# What failed and why, kept between runs so --report can say more than
+# "missing". A photo that has since arrived on disk drops out; everything
+# else carries forward until it succeeds.
+#
+def loadfailures(directory):
+    try:
+        with open(os.path.join(directory, FAILURES), encoding="utf-8") as fh:
+            held = json.load(fh)
+    except (OSError, ValueError):
+        return {}
+    return held if isinstance(held, dict) else {}
+
+def recordfailures(directory, failures):
+    held = {k: v for k, v in loadfailures(directory).items()
+            if not iscomplete(directory, k)}
+    now = utcnow()
+    for key, names, reason in failures.items:
+        held[key] = {"reason": reason, "pages": names, "when": now}
+    writejson(os.path.join(directory, FAILURES), held)
+
+
 def nodeentry(node):
     return {k: node.get(k, "") for k in ("param", "name", "url", "lat", "long")}
 
@@ -938,6 +967,36 @@ def fetchphase(conf, flickrkey):
 
 
 # immich identifies a file by the base64 of its SHA1
+#
+# Which wiki pages have no photo on disk, and the last thing that went wrong
+# fetching it. Reads the index and the directory, nothing else, and prints
+# regardless of quiet: a report nobody can see is not a report.
+#
+def report(conf):
+    directory = conf["directory"]
+    nodes = loadindex(conf["index_url"])
+    groups, rejected = groupphotos(nodes)
+    held = loadfailures(directory)
+    missing = 0
+    for key in sorted(groups):
+        if iscomplete(directory, key):
+            continue
+        missing += 1
+        record = held.get(key)
+        if record:
+            print("%s: %s (%s)" % (key, record.get("reason", "?"), record.get("when", "")[:10]))
+        else:
+            print("%s: not yet attempted" % key)
+        for node in groups[key]["nodes"]:
+            print("  %s %s" % (node["name"], node["url"]))
+    for node, reason in rejected:
+        print("%s: %s" % (node["param"], reason))
+        print("  %s %s" % (node["name"], node["url"]))
+    print("%d of %d photos missing, %d node%s with an unusable image"
+          % (missing, len(groups), len(rejected), "" if len(rejected) == 1 else "s"))
+    return 0
+
+
 def checksum(path):
     digest = hashlib.sha1()
     with open(path, "rb") as fh:
@@ -1318,6 +1377,8 @@ def summary(failures):
 
 def run(conf):
     applysettings(conf)
+    if conf["report"]:
+        return report(conf)
     failures = Failures()
     try:
         flickrkey = loadflickr(conf["flickr_auth"]) if conf["fetch"] else None
@@ -1335,13 +1396,23 @@ def run(conf):
             failures.items.extend(synced.items)
     except BackupError as e:
         error(e)
+        remember(conf, failures)
         summary(failures)
         return 1
 
+    remember(conf, failures)
     if failures or not QUIET:
         summary(failures)
     return 1 if failures else 0
 
+def remember(conf, failures):
+    if conf["dry_run"] or not os.path.isdir(conf["directory"]):
+        return
+    try:
+        recordfailures(conf["directory"], failures)
+    except OSError as e:
+        warn("%s: could not record failures: %s" % (conf["directory"], e))
+
 #
 # Under cron a traceback is a mail nobody can act on, so nothing escapes
 # here: a configuration problem exits 2, an interruption 130, and anything

test_rgl_photo_backup.py

@@ -342,6 +342,11 @@ class Settings(TestCase):
             rb.settings(args, environ={rb.ENV + "ATTEMPTS": "lots"})
         self.assertIn("ATTEMPTS", str(cm.exception))
 
+    def test_report_flag(self):
+        base = ["--settings", os.path.join(self.dir, "none"), "/d"]
+        self.assertFalse(rb.settings(rb.parseargs(base), environ={})["report"])
+        self.assertTrue(rb.settings(rb.parseargs(base + ["--report"]), environ={})["report"])
+
     def test_phase_flags(self):
         base = ["--settings", os.path.join(self.dir, "none"), "/d"]
         self.assertFalse(rb.settings(rb.parseargs(base + ["--sync-only"]), environ={})["fetch"])
@@ -653,6 +658,7 @@ class Sidecars(TestCase):
         rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg"))
         rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
         rb.writejson(os.path.join(self.dir, rb.HASHES), {})
+        rb.writejson(os.path.join(self.dir, rb.FAILURES), {})
         self.assertEqual([k for k, _ in rb.listsidecars(self.dir)], ["111", "222"])
 
     def test_utcnow_shape(self):
@@ -1408,7 +1414,7 @@ class Run(TestCase):
         conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"),
                 "immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"),
                 "index_url": "http://t/index", "fetch": True, "sync": True,
-                "dry_run": False, "limit": None, "pace": 0, "quiet": False,
+                "dry_run": False, "report": False, "limit": None, "pace": 0, "quiet": False,
                 "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False}
         conf.update(over)
         return conf
@@ -1490,3 +1496,110 @@ class Run(TestCase):
     def test_unreadable_settings_file_gives_exit_2(self):
         self.assertEqual(rb.main(["--settings", self.dir, self.dir]), 2)
         self.assertIn(self.dir, self.stderr.getvalue())
+
+
+class FailureRecords(TestCase):
+    def failures(self, *items):
+        f = rb.Failures()
+        for key, names, reason in items:
+            f.add(key, [{"name": n} for n in names], reason)
+        return f
+
+    def test_missing_file_is_empty(self):
+        self.assertEqual(rb.loadfailures(self.dir), {})
+
+    def test_records_merge_and_drop_completed(self):
+        rb.recordfailures(self.dir, self.failures(("111", ["A, E1"], "gone"),
+                                                  ("222", ["B, E2"], "HTTP 404")))
+        held = rb.loadfailures(self.dir)
+        self.assertEqual(sorted(held), ["111", "222"])
+        self.assertEqual(held["111"]["reason"], "gone")
+        self.assertEqual(held["111"]["pages"], ["A, E1"])
+        self.assertRegex(held["111"]["when"], r"^\d{4}-")
+        # 111 later succeeds; 333 newly fails; 222 untouched this run but still missing
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        rb.recordfailures(self.dir, self.failures(("333", ["C, E3"], "timeout")))
+        held = rb.loadfailures(self.dir)
+        self.assertEqual(sorted(held), ["222", "333"])
+
+    def test_unreadable_file_is_treated_as_empty(self):
+        self.write(rb.FAILURES, "{not json")
+        self.assertEqual(rb.loadfailures(self.dir), {})
+
+
+class Report(TestCase):
+    def conf(self, **over):
+        conf = {"directory": self.dir, "index_url": "http://t/index", "report": True,
+                "fetch": True, "sync": True, "dry_run": False, "limit": None, "quiet": False,
+                "flickr_auth": os.path.join(self.dir, "missing.yml"),
+                "immich_auth": os.path.join(self.dir, "missing.yml"),
+                "pace": 0, "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua",
+                "debug": False}
+        conf.update(over)
+        return conf
+
+    def index(self):
+        return json.dumps([
+            index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                        page="https://www.flickr.com/photos/x/111/"),
+            index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg",
+                        page="https://www.flickr.com/photos/x/222/"),
+            index_entry("C,_E3", "https://farm1.static.flickr.com/1/333_c_m.jpg"),
+            index_entry("D,_E4", "junk"),
+        ]).encode()
+
+    def test_lists_missing_photos_with_pages_and_reasons(self):
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        rb.writejson(os.path.join(self.dir, rb.FAILURES),
+                     {"222": {"reason": "photo 222: HTTP 404 Not Found", "pages": ["B, E2"],
+                              "when": "2026-09-13T10:00:00Z"}})
+        rb.OPEN = FakeOpen(FakeResponse(self.index()))
+        self.assertEqual(rb.run(self.conf()), 0)
+        out = self.stdout.getvalue()
+        self.assertNotIn("111", out)
+        self.assertIn("222: photo 222: HTTP 404 Not Found (2026-09-13)", out)
+        self.assertIn("  B, E2 https://london.randomness.org.uk/wiki.cgi?B%2C_E2", out)
+        self.assertIn("333: not yet attempted", out)
+        self.assertIn("  C, E3 ", out)
+        self.assertIn("D,_E4: node_image is not a url", out)
+        self.assertIn("2 of 3 photos missing, 1 node with an unusable image", out)
+        self.assertEqual(len(rb.OPEN.requests), 1)          # the index only
+
+    def test_report_prints_even_when_quiet(self):
+        rb.OPEN = FakeOpen(FakeResponse(self.index()))
+        rb.run(self.conf(quiet=True))
+        self.assertIn("3 of 3 photos missing", self.stdout.getvalue())
+
+    def test_report_needs_no_credentials(self):
+        rb.OPEN = FakeOpen(FakeResponse(b"[]"))
+        self.assertEqual(rb.run(self.conf()), 0)
+        self.assertIn("0 of 0 photos missing", self.stdout.getvalue())
+
+
+class RunRecordsFailures(TestCase):
+    def conf(self, **over):
+        conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"),
+                "immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"),
+                "index_url": "http://t/index", "fetch": True, "sync": False, "report": False,
+                "dry_run": False, "limit": None, "pace": 0, "quiet": False,
+                "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False}
+        conf.update(over)
+        return conf
+
+    def test_failed_run_writes_the_failures_file(self):
+        index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")]
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()),
+                           FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
+        self.assertEqual(rb.run(self.conf()), 1)
+        held = rb.loadfailures(self.dir)
+        self.assertIn("111", held)
+        self.assertIn("Photo not found", held["111"]["reason"])
+        self.assertEqual(held["111"]["pages"], ["A, E1"])
+
+    def test_dry_run_writes_nothing(self):
+        index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")]
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()))
+        rb.run(self.conf(dry_run=True))
+        self.assertFalse(os.path.exists(os.path.join(self.dir, rb.FAILURES)))