Record an album or tag write immich refused

immich answers a bulk album or tag write with HTTP 200 and one result per
asset, so a key without album permission produced "filing 0 asset(s)",
no recorded failure and exit 0. Refusals other than duplicate now come
back from the client and are recorded against the photo and its wiki
pages. The album and tag stages also get separate error handling, so a
failed album write no longer skips tagging for the run.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

README.md

@@ -42,11 +42,16 @@ Make the credential files `chmod 600`.
 ./rgl-photo-backup --settings other.yml DIR   # a different settings file
 ./rgl-photo-backup --debug DIR                # log every request to stderr
 ./rgl-photo-backup -q DIR                     # same as --quiet
+./rgl-photo-backup --report DIR               # which pages have no photo on disk, and why
 ```
 
 The first run downloads about 4800 photos, one Flickr info call and one
 sizes call each, so at the default one-second pace it takes a few hours.
 The pace applies to Flickr alone; Immich and the wiki index are not slowed.
+A 429 from Flickr doubles the pace for the rest of the run, up to ten seconds
+between calls, and it relaxes again after twenty untroubled requests. The
+photo that hit the limit is retried with backoff starting at fifteen seconds
+and doubling, up to eight attempts.
 Later runs skip everything already on disk, and in Immich only write an
 album membership or a description that has actually changed.
 
@@ -113,6 +118,13 @@ image URL (`url-0f4b1c2d3e4f5a6b.jpg`), with a matching sidecar. The
 directory is a complete backup on its own. A `.part` file is an interrupted
 download and is refetched next run.
 
+`.failures.json` records the last reason each photo failed, with the pages
+that use it. A photo that later arrives drops out of it. `--report` reads the
+wiki index and that file, and lists every page whose photo is not on disk
+with the reason, or `not yet attempted`, without calling Flickr or Immich, so
+it is the way to find photos that have gone from Flickr and need replacing
+on the wiki.
+
 ## Running under cron
 
 ```cron
@@ -137,11 +149,12 @@ A flag beats the environment, which beats the file, which beats the default.
 | `fetch` | `--sync-only` sets false | | `true` |
 | `sync` | `--fetch-only` sets false | | `true` |
 | `dry_run` | `--dry-run` | | `false` |
+| `report` | `--report` | | `false` |
 | `limit` | `--limit N` | | unlimited |
 | `pace` | `--pace SECONDS` | `RGL_PHOTO_BACKUP_PACE` | `1` |
 | `quiet` | `--quiet`, `-q` | | `false` |
-| `attempts` | | `RGL_PHOTO_BACKUP_ATTEMPTS` | `6` |
+| `attempts` | | `RGL_PHOTO_BACKUP_ATTEMPTS` | `8` |
-| `backoff` | | `RGL_PHOTO_BACKUP_BACKOFF` | `5` |
+| `backoff` | | `RGL_PHOTO_BACKUP_BACKOFF` | `15` |
 | `timeout` | | `RGL_PHOTO_BACKUP_TIMEOUT` | `60` |
 | `user_agent` | | `RGL_PHOTO_BACKUP_USER_AGENT` | `rgl-photo-backup/1.0` |
 | `debug` | `--debug` | `RGL_PHOTO_BACKUP_DEBUG` | `false` |

rgl-photo-backup

@@ -38,13 +38,16 @@ ALBUM          = "RGL"
 TAG_ROOT       = "rgl"
 DEVICE_ID      = "rgl-photo-backup"
 HASHES         = ".immich-hashes.json"
+FAILURES       = ".failures.json"
 
 PAGE           = 1000            # immich search page
 BATCH          = 500             # ids per album/tag write
 CHUNK          = 1024 * 1024     # hashing
 DOWNLOAD_CHUNK = 256 * 1024
 DOWNLOAD_CAP   = 100 * 1024 * 1024   # no wiki photo is this big; something is wrong
-BACKOFF_CAP    = 300.0
+BACKOFF_CAP    = 900.0
+PACE_CAP       = 10.0            # slowest a 429 storm can make flickr calls
+PACE_HOLD      = 20              # untroubled requests before the pace relaxes
 
 ENV = "RGL_PHOTO_BACKUP_"
 
@@ -57,11 +60,12 @@ DEFAULTS = {
     "fetch":       True,
     "sync":        True,
     "dry_run":     False,
+    "report":      False,
     "limit":       None,
     "pace":        1.0,
     "quiet":       False,
-    "attempts":    6,
+    "attempts":    8,
-    "backoff":     5.0,
+    "backoff":     15.0,
     "timeout":     60.0,
     "user_agent":  "rgl-photo-backup/1.0",
     "debug":       False,
@@ -72,6 +76,8 @@ USER_AGENT   = os.environ.get(ENV + "USER_AGENT", DEFAULTS["user_agent"])
 ATTEMPTS     = int(os.environ.get(ENV + "ATTEMPTS", DEFAULTS["attempts"]))
 BACKOFF_BASE = float(os.environ.get(ENV + "BACKOFF", DEFAULTS["backoff"]))
 PACE         = float(os.environ.get(ENV + "PACE", DEFAULTS["pace"]))
+PACE_FLOOR   = PACE              # the configured pace; PACE rises above it after a 429
+PACEHELD     = 0
 DEBUG        = bool(os.environ.get(ENV + "DEBUG"))
 QUIET        = False
 
@@ -154,6 +160,7 @@ SETTING_KINDS = {
     "fetch":       bool,
     "sync":        bool,
     "dry_run":     bool,
+    "report":      bool,
     "limit":       int,
     "pace":        float,
     "quiet":       bool,
@@ -205,6 +212,9 @@ def parseargs(argv=None):
                         help="disk to immich, no RGL or flickr calls")
     parser.add_argument("--dry-run", action="store_true", default=None,
                         help="report what would be fetched and written, change nothing")
+    parser.add_argument("--report", action="store_true", default=None,
+                        help="list the wiki photos not on disk, with their pages and "
+                             "the last recorded reason; reads the index only")
     parser.add_argument("--limit", type=int, metavar="N",
                         help="stop after N photos, for a trial run")
     parser.add_argument("--pace", type=float, metavar="SECONDS",
@@ -253,6 +263,7 @@ def settings(args, environ=None):
         "fetch":       pick("fetch", False if args.sync_only else None, DEFAULTS["fetch"]),
         "sync":        pick("sync", False if args.fetch_only else None, DEFAULTS["sync"]),
         "dry_run":     pick("dry_run", args.dry_run, DEFAULTS["dry_run"]),
+        "report":      pick("report", args.report, DEFAULTS["report"]),
         "limit":       pick("limit", args.limit, DEFAULTS["limit"]),
         "pace":        pick("pace", args.pace, DEFAULTS["pace"]),
         "quiet":       pick("quiet", args.quiet, DEFAULTS["quiet"]),
@@ -303,9 +314,12 @@ def backoff(attempt):
 # failures. consume() is handed the open response so a body can be read into
 # memory or streamed to disk under one retry policy. 429 and 5xx are worth
 # another go; any other 4xx is our fault and will not improve. Only flickr
-# needs pacing, and only flickr's limit is worth slowing the run for.
+# needs pacing, and only flickr's limit is worth slowing the run for: a 429
+# there is an hourly window, which no single photo's backoff can wait out, so
+# it slows every later paced request and relaxes only after a quiet spell.
 #
 def retrying(url, context, consume, headers=None, data=None, method="GET", pace=False):
+    global PACE, PACEHELD
     sent = {"User-Agent": USER_AGENT}
     sent.update(headers or {})
     for attempt in range(1, ATTEMPTS + 1):
@@ -317,7 +331,7 @@ def retrying(url, context, consume, headers=None, data=None, method="GET", pace=
         try:
             response = OPEN(request, timeout=HTTP_TIMEOUT)
             try:
-                return consume(response)
+                result = consume(response)
             finally:
                 response.close()
         except BackupError:
@@ -332,6 +346,10 @@ def retrying(url, context, consume, headers=None, data=None, method="GET", pace=
                 e.close()
                 raise HTTPFailure("%s: HTTP %s %s%s" % (context, e.code, e.reason, detail),
                                   code=e.code)
+            if e.code == 429 and pace:
+                PACE = min(max(PACE * 2, 1.0), PACE_CAP)
+                PACEHELD = PACE_HOLD
+                warn("%s: rate limited, pacing flickr requests %.1fs apart" % (context, PACE))
             wait = retryafter(e)
             reason = "HTTP %s %s" % (e.code, e.reason)
             code = e.code
@@ -340,6 +358,13 @@ def retrying(url, context, consume, headers=None, data=None, method="GET", pace=
             wait = None
             reason = "%s: %s" % (type(e).__name__, e)
             code = None
+        else:
+            if pace and PACE > PACE_FLOOR:
+                if PACEHELD > 0:
+                    PACEHELD -= 1
+                else:
+                    PACE = max(PACE_FLOOR, PACE * 0.9)
+            return result
 
         if attempt == ATTEMPTS:
             raise HTTPFailure("%s: giving up after %d attempts (%s)"
@@ -647,7 +672,7 @@ def listsidecars(directory):
     except OSError as e:
         raise BackupError("%s: cannot be listed: %s" % (directory, e.strerror or e))
     for name in names:
-        if not name.endswith(".json") or name == HASHES:
+        if not name.endswith(".json") or name in (HASHES, FAILURES):
             continue
         side = readsidecar(os.path.join(directory, name))
         if side:
@@ -793,6 +818,28 @@ class Failures:
         return len(self.items)
 
 
+#
+# What failed and why, kept between runs so --report can say more than
+# "missing". A photo that has since arrived on disk drops out; everything
+# else carries forward until it succeeds.
+#
+def loadfailures(directory):
+    try:
+        with open(os.path.join(directory, FAILURES), encoding="utf-8") as fh:
+            held = json.load(fh)
+    except (OSError, ValueError):
+        return {}
+    return held if isinstance(held, dict) else {}
+
+def recordfailures(directory, failures):
+    held = {k: v for k, v in loadfailures(directory).items()
+            if not iscomplete(directory, k)}
+    now = utcnow()
+    for key, names, reason in failures.items:
+        held[key] = {"reason": reason, "pages": names, "when": now}
+    writejson(os.path.join(directory, FAILURES), held)
+
+
 def nodeentry(node):
     return {k: node.get(k, "") for k in ("param", "name", "url", "lat", "long")}
 
@@ -920,6 +967,36 @@ def fetchphase(conf, flickrkey):
 
 
 # immich identifies a file by the base64 of its SHA1
+#
+# Which wiki pages have no photo on disk, and the last thing that went wrong
+# fetching it. Reads the index and the directory, nothing else, and prints
+# regardless of quiet: a report nobody can see is not a report.
+#
+def report(conf):
+    directory = conf["directory"]
+    nodes = loadindex(conf["index_url"])
+    groups, rejected = groupphotos(nodes)
+    held = loadfailures(directory)
+    missing = 0
+    for key in sorted(groups):
+        if iscomplete(directory, key):
+            continue
+        missing += 1
+        record = held.get(key)
+        if record:
+            print("%s: %s (%s)" % (key, record.get("reason", "?"), record.get("when", "")[:10]))
+        else:
+            print("%s: not yet attempted" % key)
+        for node in groups[key]["nodes"]:
+            print("  %s %s" % (node["name"], node["url"]))
+    for node, reason in rejected:
+        print("%s: %s" % (node["param"], reason))
+        print("  %s %s" % (node["name"], node["url"]))
+    print("%d of %d photos missing, %d node%s with an unusable image"
+          % (missing, len(groups), len(rejected), "" if len(rejected) == 1 else "s"))
+    return 0
+
+
 def checksum(path):
     digest = hashlib.sha1()
     with open(path, "rb") as fh:
@@ -1089,13 +1166,23 @@ class Immich:
         album = self.call("GET", "/albums/%s" % albumid) or {}
         return {a["id"] for a in album.get("assets", [])}
 
+    #
+    # immich answers a bulk write with one result per id. A duplicate is what
+    # a second run should see; any other refusal (no_permission, not_found)
+    # is handed back so the caller can record it against the photo, since
+    # the request as a whole came back 200.
+    #
     def addtoalbum(self, albumid, ids):
-        added = 0
+        added, refused = 0, []
         for start in range(0, len(ids), BATCH):
             results = self.call("PUT", "/albums/%s/assets" % albumid,
                                 {"ids": ids[start:start + BATCH]}) or []
-            added += sum(1 for r in results if r.get("success"))
+            for r in results:
-        return added
+                if r.get("success"):
+                    added += 1
+                elif r.get("error") != "duplicate":
+                    refused.append((r.get("id"), r.get("error") or "refused"))
+        return (added, refused)
 
     # Albums shared with this user are listed too, and one of those could
     # carry the RGL name, so only the ones this user owns count.
@@ -1121,12 +1208,18 @@ class Immich:
     # is what a second run should see and not a failure
     def tagassets(self, tagid, ids):
         tagged = already = 0
+        refused = []
         for start in range(0, len(ids), BATCH):
             results = self.call("PUT", "/tags/%s/assets" % tagid,
                                 {"ids": ids[start:start + BATCH]}) or []
-            tagged += sum(1 for r in results if r.get("success"))
+            for r in results:
-            already += sum(1 for r in results if r.get("error") == "duplicate")
+                if r.get("success"):
-        return (tagged, already)
+                    tagged += 1
+                elif r.get("error") == "duplicate":
+                    already += 1
+                else:
+                    refused.append((r.get("id"), r.get("error") or "refused"))
+        return (tagged, already, refused)
 
 
 def isoutc(epoch):
@@ -1249,8 +1342,16 @@ def syncphase(conf, immich):
         inalbum = immich.albumassets(albumid)
         missing = [a for a in matched if a not in inalbum]
         if missing:
-            say("album %s: filing %d asset(s)" % (ALBUM, immich.addtoalbum(albumid, missing)))
+            added, refused = immich.addtoalbum(albumid, missing)
+            say("album %s: filing %d asset(s)" % (ALBUM, added))
+            for assetid, why in refused:
+                side = matched.get(assetid, {})
+                failures.add(side.get("key", assetid), sidenodes(side), "album %s: %s" % (ALBUM, why))
+    except BackupError as e:
+        failures.add("album", [], e)
 
+    # Tagging gets its own try so a refused album does not also cost the tags
+    try:
         wanted = {}
         for assetid, side in matched.items():
             for tag in tagsfor(side):
@@ -1261,11 +1362,14 @@ def syncphase(conf, immich):
                 if tag not in ids:
                     failures.add(tag, [], "immich returned no id for this tag")
                     continue
-                tagged, already = immich.tagassets(ids[tag], wanted[tag])
+                tagged, already, refused = immich.tagassets(ids[tag], wanted[tag])
                 if tagged:
                     say("tag %s: %d tagged, %d already" % (tag, tagged, already))
+                for assetid, why in refused:
+                    side = matched.get(assetid, {})
+                    failures.add(side.get("key", assetid), sidenodes(side), "tag %s: %s" % (tag, why))
     except BackupError as e:
-        failures.add("album/tags", [], e)
+        failures.add("tags", [], e)
 
     for assetid, side in fresh.items():
         try:
@@ -1279,8 +1383,9 @@ def syncphase(conf, immich):
 
 
 def applysettings(conf):
-    global PACE, ATTEMPTS, BACKOFF_BASE, HTTP_TIMEOUT, USER_AGENT, QUIET, DEBUG
+    global PACE, PACE_FLOOR, PACEHELD, ATTEMPTS, BACKOFF_BASE, HTTP_TIMEOUT, USER_AGENT, QUIET, DEBUG
-    PACE         = float(conf["pace"])
+    PACE         = PACE_FLOOR = float(conf["pace"])
+    PACEHELD     = 0
     ATTEMPTS     = int(conf["attempts"])
     BACKOFF_BASE = float(conf["backoff"])
     HTTP_TIMEOUT = float(conf["timeout"])
@@ -1299,6 +1404,8 @@ def summary(failures):
 
 def run(conf):
     applysettings(conf)
+    if conf["report"]:
+        return report(conf)
     failures = Failures()
     try:
         flickrkey = loadflickr(conf["flickr_auth"]) if conf["fetch"] else None
@@ -1316,13 +1423,23 @@ def run(conf):
             failures.items.extend(synced.items)
     except BackupError as e:
         error(e)
+        remember(conf, failures)
         summary(failures)
         return 1
 
+    remember(conf, failures)
     if failures or not QUIET:
         summary(failures)
     return 1 if failures else 0
 
+def remember(conf, failures):
+    if conf["dry_run"] or not os.path.isdir(conf["directory"]):
+        return
+    try:
+        recordfailures(conf["directory"], failures)
+    except OSError as e:
+        warn("%s: could not record failures: %s" % (conf["directory"], e))
+
 #
 # Under cron a traceback is a mail nobody can act on, so nothing escapes
 # here: a configuration problem exits 2, an interruption 130, and anything

test_rgl_photo_backup.py

@@ -74,8 +74,9 @@ class FakeOpen:
 class TestCase(unittest.TestCase):
     """Silences output and restores the module globals tests move around."""
 
-    GLOBALS = ("OPEN", "SLEEP", "ATTEMPTS", "BACKOFF_BASE", "PACE", "QUIET",
+    GLOBALS = ("OPEN", "SLEEP", "ATTEMPTS", "BACKOFF_BASE", "PACE", "PACE_FLOOR",
-               "DEBUG", "HTTP_TIMEOUT", "USER_AGENT", "STATS", "DOWNLOAD_CAP")
+               "PACEHELD", "QUIET", "DEBUG", "HTTP_TIMEOUT", "USER_AGENT", "STATS",
+               "DOWNLOAD_CAP")
 
     def setUp(self):
         self._output = contextlib.ExitStack()
@@ -87,7 +88,8 @@ class TestCase(unittest.TestCase):
         self.addCleanup(self._output.close)
         self._saved = {k: getattr(rb, k) for k in self.GLOBALS}
         rb.SLEEP = lambda seconds: None
-        rb.PACE = 0.0
+        rb.PACE = rb.PACE_FLOOR = 0.0
+        rb.PACEHELD = 0
         rb.STATS = dict.fromkeys(rb.STATS, 0)
         self.tmp = tempfile.TemporaryDirectory()
         self.addCleanup(self.tmp.cleanup)
@@ -309,7 +311,9 @@ class Settings(TestCase):
         self.assertTrue(conf["fetch"] and conf["sync"])
         self.assertFalse(conf["dry_run"])
         self.assertIsNone(conf["limit"])
-        self.assertEqual(conf["attempts"], 6)
+        self.assertEqual(conf["attempts"], 8)
+        self.assertEqual(conf["backoff"], 15.0)
+        self.assertEqual(rb.BACKOFF_CAP, 900.0)
         self.assertEqual(conf["flickr_auth"], os.path.expanduser(rb.FLICKR_CONFIG))
 
     def test_defaults_are_literals_not_moving_constants(self):
@@ -319,7 +323,7 @@ class Settings(TestCase):
         self.assertEqual(conf["pace"], rb.DEFAULTS["pace"])
         self.assertEqual(conf["attempts"], rb.DEFAULTS["attempts"])
         self.assertEqual(conf["user_agent"], rb.DEFAULTS["user_agent"])
-        self.assertEqual((rb.DEFAULTS["pace"], rb.DEFAULTS["attempts"]), (1.0, 6))
+        self.assertEqual((rb.DEFAULTS["pace"], rb.DEFAULTS["attempts"]), (1.0, 8))
 
     def test_debug_is_a_setting(self):
         path = self.write("s.yml", "directory: /d\ndebug: true\n")
@@ -338,6 +342,11 @@ class Settings(TestCase):
             rb.settings(args, environ={rb.ENV + "ATTEMPTS": "lots"})
         self.assertIn("ATTEMPTS", str(cm.exception))
 
+    def test_report_flag(self):
+        base = ["--settings", os.path.join(self.dir, "none"), "/d"]
+        self.assertFalse(rb.settings(rb.parseargs(base), environ={})["report"])
+        self.assertTrue(rb.settings(rb.parseargs(base + ["--report"]), environ={})["report"])
+
     def test_phase_flags(self):
         base = ["--settings", os.path.join(self.dir, "none"), "/d"]
         self.assertFalse(rb.settings(rb.parseargs(base + ["--sync-only"]), environ={})["fetch"])
@@ -430,6 +439,39 @@ class Retrying(TestCase):
         rb.flickrcall("flickr.test.echo", {}, "KEY")
         self.assertEqual(waits, [2.0])
 
+    def test_429_slows_every_later_flickr_call(self):
+        waits = []
+        rb.SLEEP = waits.append
+        rb.PACE = rb.PACE_FLOOR = 1.0
+        rb.OPEN = FakeOpen(http_error(429), FakeResponse({"stat": "ok"}),
+                           FakeResponse({"stat": "ok"}))
+        rb.flickrcall("flickr.test.echo", {}, "KEY")
+        self.assertEqual(rb.PACE, 2.0)
+        rb.flickrcall("flickr.test.echo", {}, "KEY")
+        self.assertEqual(waits[-1], 2.0)
+        self.assertEqual(rb.PACEHELD, rb.PACE_HOLD - 2)
+        self.assertIn("pacing", self.stderr.getvalue())
+
+    def test_pace_relaxes_back_to_the_configured_floor(self):
+        rb.PACE_FLOOR = 1.0
+        rb.PACE = 4.0
+        rb.OPEN = FakeOpen(*[FakeResponse({"stat": "ok"}) for _ in range(40)])
+        for _ in range(40):
+            rb.flickrcall("flickr.test.echo", {}, "KEY")
+        self.assertEqual(rb.PACE, 1.0)
+
+    def test_pace_never_exceeds_the_cap(self):
+        rb.PACE = rb.PACE_FLOOR = 8.0
+        rb.OPEN = FakeOpen(http_error(429), FakeResponse({"stat": "ok"}))
+        rb.flickrcall("flickr.test.echo", {}, "KEY")
+        self.assertEqual(rb.PACE, rb.PACE_CAP)
+
+    def test_429_from_immich_leaves_the_pace_alone(self):
+        rb.PACE = rb.PACE_FLOOR = 1.0
+        rb.OPEN = FakeOpen(http_error(429), FakeResponse({"id": "u1"}))
+        rb.Immich("https://i/api", "k").me()
+        self.assertEqual(rb.PACE, 1.0)
+
     def test_pace_leaves_immich_and_the_index_alone(self):
         waits = []
         rb.SLEEP = waits.append
@@ -616,6 +658,7 @@ class Sidecars(TestCase):
         rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg"))
         rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
         rb.writejson(os.path.join(self.dir, rb.HASHES), {})
+        rb.writejson(os.path.join(self.dir, rb.FAILURES), {})
         self.assertEqual([k for k, _ in rb.listsidecars(self.dir)], ["111", "222"])
 
     def test_utcnow_shape(self):
@@ -1076,13 +1119,13 @@ class ImmichClient(TestCase):
         self.assertEqual(im.albums("u1"), {"RGL": "al1"})
         self.assertEqual(im.createalbum("X"), "al2")
         self.assertEqual(im.albumassets("al1"), {"a1", "a2"})
-        self.assertEqual(im.addtoalbum("al1", ["a3", "a1"]), 1)
+        self.assertEqual(im.addtoalbum("al1", ["a3", "a1"]), (1, []))
 
     def test_tags(self):
         im = self.client(FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/x", "id": "t2"}]),
                          FakeResponse([{"success": True}, {"success": False, "error": "duplicate"}]))
         self.assertEqual(im.tagids(["rgl", "rgl/licence/x"]), {"rgl": "t1", "rgl/licence/x": "t2"})
-        self.assertEqual(im.tagassets("t1", ["a1", "a2"]), (1, 1))
+        self.assertEqual(im.tagassets("t1", ["a1", "a2"]), (1, 1, []))
 
     def test_tagids_skips_malformed_entries(self):
         im = self.client(FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/broken"}]))
@@ -1093,10 +1136,30 @@ class ImmichClient(TestCase):
         im = self.client(FakeResponse([{"success": True}] * 500),
                          FakeResponse([{"success": True}] * 500),
                          FakeResponse([{"success": True}] * 200))
-        self.assertEqual(im.addtoalbum("al", ids), 1200)
+        self.assertEqual(im.addtoalbum("al", ids), (1200, []))
         self.assertEqual(len(rb.OPEN.requests), 3)
 
 
+class ImmichRefusals(TestCase):
+    def client(self, *responses):
+        rb.OPEN = FakeOpen(*responses)
+        return rb.Immich("https://immich.example/api", "KEY")
+
+    def test_album_refusals_are_returned_not_dropped(self):
+        im = self.client(FakeResponse([{"id": "a1", "success": False, "error": "no_permission"},
+                                       {"id": "a2", "success": False, "error": "duplicate"},
+                                       {"id": "a3", "success": True}]))
+        self.assertEqual(im.addtoalbum("al1", ["a1", "a2", "a3"]),
+                         (1, [("a1", "no_permission")]))
+
+    def test_tag_refusals_are_returned_not_dropped(self):
+        im = self.client(FakeResponse([{"id": "a1", "success": False, "error": "not_found"},
+                                       {"id": "a2", "success": False, "error": "duplicate"},
+                                       {"id": "a3", "success": True}]))
+        self.assertEqual(im.tagassets("t1", ["a1", "a2", "a3"]),
+                         (1, 1, [("a1", "not_found")]))
+
+
 class Hashing(TestCase):
     def test_checksum_is_base64_sha1(self):
         path = self.write("f", b"hello")
@@ -1358,12 +1421,43 @@ class SyncPhase(TestCase):
             FakeResponse({"assets": {"items": [], "nextPage": None}}),
             FakeResponse({"id": "a1", "status": "created"}, status=201),   # upload 111
             http_error(500),                                              # albums
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"}]),   # tags still run
+            FakeResponse([{"success": True}]), FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
             FakeResponse({"id": "a1", "exifInfo": {}}),                   # re-read a1
             FakeResponse(b""))                                            # update a1
         failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k"))
         self.assertEqual(len(failures), 1)
-        reason = failures.items[0][2]
+        key, names, reason = failures.items[0]
+        self.assertEqual(key, "album")
         self.assertTrue("albums" in reason or "HTTP 500" in reason)
+        self.assertEqual(rb.OPEN.requests[4].full_url, "https://i/api/tags")
+
+    def test_album_refusal_is_recorded_against_the_photo(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "created"}, status=201),   # upload 111
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": []}),
+            FakeResponse([{"id": "a1", "success": False, "error": "no_permission"}]),
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"}]),
+            FakeResponse([{"id": "a1", "success": False, "error": "no_permission"}]),
+            FakeResponse([{"success": True}]), FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a1", "exifInfo": {}}),
+            FakeResponse(b""))
+        failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 2)
+        (k1, names1, r1), (k2, names2, r2) = failures.items
+        self.assertEqual((k1, names1), ("111", ["A, E1"]))
+        self.assertIn("album RGL: no_permission", r1)
+        self.assertEqual((k2, names2), ("111", ["A, E1"]))
+        self.assertIn("tag rgl: no_permission", r2)
+        self.assertIn("no_permission", self.stderr.getvalue())
 
 
 class Run(TestCase):
@@ -1371,7 +1465,7 @@ class Run(TestCase):
         conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"),
                 "immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"),
                 "index_url": "http://t/index", "fetch": True, "sync": True,
-                "dry_run": False, "limit": None, "pace": 0, "quiet": False,
+                "dry_run": False, "report": False, "limit": None, "pace": 0, "quiet": False,
                 "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False}
         conf.update(over)
         return conf
@@ -1453,3 +1547,110 @@ class Run(TestCase):
     def test_unreadable_settings_file_gives_exit_2(self):
         self.assertEqual(rb.main(["--settings", self.dir, self.dir]), 2)
         self.assertIn(self.dir, self.stderr.getvalue())
+
+
+class FailureRecords(TestCase):
+    def failures(self, *items):
+        f = rb.Failures()
+        for key, names, reason in items:
+            f.add(key, [{"name": n} for n in names], reason)
+        return f
+
+    def test_missing_file_is_empty(self):
+        self.assertEqual(rb.loadfailures(self.dir), {})
+
+    def test_records_merge_and_drop_completed(self):
+        rb.recordfailures(self.dir, self.failures(("111", ["A, E1"], "gone"),
+                                                  ("222", ["B, E2"], "HTTP 404")))
+        held = rb.loadfailures(self.dir)
+        self.assertEqual(sorted(held), ["111", "222"])
+        self.assertEqual(held["111"]["reason"], "gone")
+        self.assertEqual(held["111"]["pages"], ["A, E1"])
+        self.assertRegex(held["111"]["when"], r"^\d{4}-")
+        # 111 later succeeds; 333 newly fails; 222 untouched this run but still missing
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        rb.recordfailures(self.dir, self.failures(("333", ["C, E3"], "timeout")))
+        held = rb.loadfailures(self.dir)
+        self.assertEqual(sorted(held), ["222", "333"])
+
+    def test_unreadable_file_is_treated_as_empty(self):
+        self.write(rb.FAILURES, "{not json")
+        self.assertEqual(rb.loadfailures(self.dir), {})
+
+
+class Report(TestCase):
+    def conf(self, **over):
+        conf = {"directory": self.dir, "index_url": "http://t/index", "report": True,
+                "fetch": True, "sync": True, "dry_run": False, "limit": None, "quiet": False,
+                "flickr_auth": os.path.join(self.dir, "missing.yml"),
+                "immich_auth": os.path.join(self.dir, "missing.yml"),
+                "pace": 0, "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua",
+                "debug": False}
+        conf.update(over)
+        return conf
+
+    def index(self):
+        return json.dumps([
+            index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                        page="https://www.flickr.com/photos/x/111/"),
+            index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg",
+                        page="https://www.flickr.com/photos/x/222/"),
+            index_entry("C,_E3", "https://farm1.static.flickr.com/1/333_c_m.jpg"),
+            index_entry("D,_E4", "junk"),
+        ]).encode()
+
+    def test_lists_missing_photos_with_pages_and_reasons(self):
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"x")
+        rb.writejson(os.path.join(self.dir, rb.FAILURES),
+                     {"222": {"reason": "photo 222: HTTP 404 Not Found", "pages": ["B, E2"],
+                              "when": "2026-09-13T10:00:00Z"}})
+        rb.OPEN = FakeOpen(FakeResponse(self.index()))
+        self.assertEqual(rb.run(self.conf()), 0)
+        out = self.stdout.getvalue()
+        self.assertNotIn("111", out)
+        self.assertIn("222: photo 222: HTTP 404 Not Found (2026-09-13)", out)
+        self.assertIn("  B, E2 https://london.randomness.org.uk/wiki.cgi?B%2C_E2", out)
+        self.assertIn("333: not yet attempted", out)
+        self.assertIn("  C, E3 ", out)
+        self.assertIn("D,_E4: node_image is not a url", out)
+        self.assertIn("2 of 3 photos missing, 1 node with an unusable image", out)
+        self.assertEqual(len(rb.OPEN.requests), 1)          # the index only
+
+    def test_report_prints_even_when_quiet(self):
+        rb.OPEN = FakeOpen(FakeResponse(self.index()))
+        rb.run(self.conf(quiet=True))
+        self.assertIn("3 of 3 photos missing", self.stdout.getvalue())
+
+    def test_report_needs_no_credentials(self):
+        rb.OPEN = FakeOpen(FakeResponse(b"[]"))
+        self.assertEqual(rb.run(self.conf()), 0)
+        self.assertIn("0 of 0 photos missing", self.stdout.getvalue())
+
+
+class RunRecordsFailures(TestCase):
+    def conf(self, **over):
+        conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"),
+                "immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"),
+                "index_url": "http://t/index", "fetch": True, "sync": False, "report": False,
+                "dry_run": False, "limit": None, "pace": 0, "quiet": False,
+                "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False}
+        conf.update(over)
+        return conf
+
+    def test_failed_run_writes_the_failures_file(self):
+        index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")]
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()),
+                           FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"}))
+        self.assertEqual(rb.run(self.conf()), 1)
+        held = rb.loadfailures(self.dir)
+        self.assertIn("111", held)
+        self.assertIn("Photo not found", held["111"]["reason"])
+        self.assertEqual(held["111"]["pages"], ["A, E1"])
+
+    def test_dry_run_writes_nothing(self):
+        index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")]
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()))
+        rb.run(self.conf(dry_run=True))
+        self.assertFalse(os.path.exists(os.path.join(self.dir, rb.FAILURES)))