diff --git a/README.md b/README.md index d2a522c..b801dc7 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Make the credential files `chmod 600`. ./rgl-photo-backup --settings other.yml DIR # a different settings file ./rgl-photo-backup --debug DIR # log every request to stderr ./rgl-photo-backup -q DIR # same as --quiet +./rgl-photo-backup --report DIR # which pages have no photo on disk, and why ``` The first run downloads about 4800 photos, one Flickr info call and one @@ -117,6 +118,13 @@ image URL (`url-0f4b1c2d3e4f5a6b.jpg`), with a matching sidecar. The directory is a complete backup on its own. A `.part` file is an interrupted download and is refetched next run. +`.failures.json` records the last reason each photo failed, with the pages +that use it. A photo that later arrives drops out of it. `--report` reads the +wiki index and that file, and lists every page whose photo is not on disk +with the reason, or `not yet attempted`, without calling Flickr or Immich, so +it is the way to find photos that have gone from Flickr and need replacing +on the wiki. + ## Running under cron ```cron @@ -141,6 +149,7 @@ A flag beats the environment, which beats the file, which beats the default. | `fetch` | `--sync-only` sets false | | `true` | | `sync` | `--fetch-only` sets false | | `true` | | `dry_run` | `--dry-run` | | `false` | +| `report` | `--report` | | `false` | | `limit` | `--limit N` | | unlimited | | `pace` | `--pace SECONDS` | `RGL_PHOTO_BACKUP_PACE` | `1` | | `quiet` | `--quiet`, `-q` | | `false` | diff --git a/rgl-photo-backup b/rgl-photo-backup index cd29c38..06e368b 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -38,6 +38,7 @@ ALBUM = "RGL" TAG_ROOT = "rgl" DEVICE_ID = "rgl-photo-backup" HASHES = ".immich-hashes.json" +FAILURES = ".failures.json" PAGE = 1000 # immich search page BATCH = 500 # ids per album/tag write @@ -59,6 +60,7 @@ DEFAULTS = { "fetch": True, "sync": True, "dry_run": False, + "report": False, "limit": None, "pace": 1.0, "quiet": False, @@ -158,6 +160,7 @@ SETTING_KINDS = { "fetch": bool, "sync": bool, "dry_run": bool, + "report": bool, "limit": int, "pace": float, "quiet": bool, @@ -209,6 +212,9 @@ def parseargs(argv=None): help="disk to immich, no RGL or flickr calls") parser.add_argument("--dry-run", action="store_true", default=None, help="report what would be fetched and written, change nothing") + parser.add_argument("--report", action="store_true", default=None, + help="list the wiki photos not on disk, with their pages and " + "the last recorded reason; reads the index only") parser.add_argument("--limit", type=int, metavar="N", help="stop after N photos, for a trial run") parser.add_argument("--pace", type=float, metavar="SECONDS", @@ -257,6 +263,7 @@ def settings(args, environ=None): "fetch": pick("fetch", False if args.sync_only else None, DEFAULTS["fetch"]), "sync": pick("sync", False if args.fetch_only else None, DEFAULTS["sync"]), "dry_run": pick("dry_run", args.dry_run, DEFAULTS["dry_run"]), + "report": pick("report", args.report, DEFAULTS["report"]), "limit": pick("limit", args.limit, DEFAULTS["limit"]), "pace": pick("pace", args.pace, DEFAULTS["pace"]), "quiet": pick("quiet", args.quiet, DEFAULTS["quiet"]), @@ -665,7 +672,7 @@ def listsidecars(directory): except OSError as e: raise BackupError("%s: cannot be listed: %s" % (directory, e.strerror or e)) for name in names: - if not name.endswith(".json") or name == HASHES: + if not name.endswith(".json") or name in (HASHES, FAILURES): continue side = readsidecar(os.path.join(directory, name)) if side: @@ -811,6 +818,28 @@ class Failures: return len(self.items) +# +# What failed and why, kept between runs so --report can say more than +# "missing". A photo that has since arrived on disk drops out; everything +# else carries forward until it succeeds. +# +def loadfailures(directory): + try: + with open(os.path.join(directory, FAILURES), encoding="utf-8") as fh: + held = json.load(fh) + except (OSError, ValueError): + return {} + return held if isinstance(held, dict) else {} + +def recordfailures(directory, failures): + held = {k: v for k, v in loadfailures(directory).items() + if not iscomplete(directory, k)} + now = utcnow() + for key, names, reason in failures.items: + held[key] = {"reason": reason, "pages": names, "when": now} + writejson(os.path.join(directory, FAILURES), held) + + def nodeentry(node): return {k: node.get(k, "") for k in ("param", "name", "url", "lat", "long")} @@ -938,6 +967,36 @@ def fetchphase(conf, flickrkey): # immich identifies a file by the base64 of its SHA1 +# +# Which wiki pages have no photo on disk, and the last thing that went wrong +# fetching it. Reads the index and the directory, nothing else, and prints +# regardless of quiet: a report nobody can see is not a report. +# +def report(conf): + directory = conf["directory"] + nodes = loadindex(conf["index_url"]) + groups, rejected = groupphotos(nodes) + held = loadfailures(directory) + missing = 0 + for key in sorted(groups): + if iscomplete(directory, key): + continue + missing += 1 + record = held.get(key) + if record: + print("%s: %s (%s)" % (key, record.get("reason", "?"), record.get("when", "")[:10])) + else: + print("%s: not yet attempted" % key) + for node in groups[key]["nodes"]: + print(" %s %s" % (node["name"], node["url"])) + for node, reason in rejected: + print("%s: %s" % (node["param"], reason)) + print(" %s %s" % (node["name"], node["url"])) + print("%d of %d photos missing, %d node%s with an unusable image" + % (missing, len(groups), len(rejected), "" if len(rejected) == 1 else "s")) + return 0 + + def checksum(path): digest = hashlib.sha1() with open(path, "rb") as fh: @@ -1318,6 +1377,8 @@ def summary(failures): def run(conf): applysettings(conf) + if conf["report"]: + return report(conf) failures = Failures() try: flickrkey = loadflickr(conf["flickr_auth"]) if conf["fetch"] else None @@ -1335,13 +1396,23 @@ def run(conf): failures.items.extend(synced.items) except BackupError as e: error(e) + remember(conf, failures) summary(failures) return 1 + remember(conf, failures) if failures or not QUIET: summary(failures) return 1 if failures else 0 +def remember(conf, failures): + if conf["dry_run"] or not os.path.isdir(conf["directory"]): + return + try: + recordfailures(conf["directory"], failures) + except OSError as e: + warn("%s: could not record failures: %s" % (conf["directory"], e)) + # # Under cron a traceback is a mail nobody can act on, so nothing escapes # here: a configuration problem exits 2, an interruption 130, and anything diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index 67d8143..3fc7edc 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -342,6 +342,11 @@ class Settings(TestCase): rb.settings(args, environ={rb.ENV + "ATTEMPTS": "lots"}) self.assertIn("ATTEMPTS", str(cm.exception)) + def test_report_flag(self): + base = ["--settings", os.path.join(self.dir, "none"), "/d"] + self.assertFalse(rb.settings(rb.parseargs(base), environ={})["report"]) + self.assertTrue(rb.settings(rb.parseargs(base + ["--report"]), environ={})["report"]) + def test_phase_flags(self): base = ["--settings", os.path.join(self.dir, "none"), "/d"] self.assertFalse(rb.settings(rb.parseargs(base + ["--sync-only"]), environ={})["fetch"]) @@ -653,6 +658,7 @@ class Sidecars(TestCase): rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg")) rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) rb.writejson(os.path.join(self.dir, rb.HASHES), {}) + rb.writejson(os.path.join(self.dir, rb.FAILURES), {}) self.assertEqual([k for k, _ in rb.listsidecars(self.dir)], ["111", "222"]) def test_utcnow_shape(self): @@ -1408,7 +1414,7 @@ class Run(TestCase): conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"), "immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"), "index_url": "http://t/index", "fetch": True, "sync": True, - "dry_run": False, "limit": None, "pace": 0, "quiet": False, + "dry_run": False, "report": False, "limit": None, "pace": 0, "quiet": False, "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False} conf.update(over) return conf @@ -1490,3 +1496,110 @@ class Run(TestCase): def test_unreadable_settings_file_gives_exit_2(self): self.assertEqual(rb.main(["--settings", self.dir, self.dir]), 2) self.assertIn(self.dir, self.stderr.getvalue()) + + +class FailureRecords(TestCase): + def failures(self, *items): + f = rb.Failures() + for key, names, reason in items: + f.add(key, [{"name": n} for n in names], reason) + return f + + def test_missing_file_is_empty(self): + self.assertEqual(rb.loadfailures(self.dir), {}) + + def test_records_merge_and_drop_completed(self): + rb.recordfailures(self.dir, self.failures(("111", ["A, E1"], "gone"), + ("222", ["B, E2"], "HTTP 404"))) + held = rb.loadfailures(self.dir) + self.assertEqual(sorted(held), ["111", "222"]) + self.assertEqual(held["111"]["reason"], "gone") + self.assertEqual(held["111"]["pages"], ["A, E1"]) + self.assertRegex(held["111"]["when"], r"^\d{4}-") + # 111 later succeeds; 333 newly fails; 222 untouched this run but still missing + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + rb.recordfailures(self.dir, self.failures(("333", ["C, E3"], "timeout"))) + held = rb.loadfailures(self.dir) + self.assertEqual(sorted(held), ["222", "333"]) + + def test_unreadable_file_is_treated_as_empty(self): + self.write(rb.FAILURES, "{not json") + self.assertEqual(rb.loadfailures(self.dir), {}) + + +class Report(TestCase): + def conf(self, **over): + conf = {"directory": self.dir, "index_url": "http://t/index", "report": True, + "fetch": True, "sync": True, "dry_run": False, "limit": None, "quiet": False, + "flickr_auth": os.path.join(self.dir, "missing.yml"), + "immich_auth": os.path.join(self.dir, "missing.yml"), + "pace": 0, "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", + "debug": False} + conf.update(over) + return conf + + def index(self): + return json.dumps([ + index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg", + page="https://www.flickr.com/photos/x/111/"), + index_entry("B,_E2", "https://farm1.static.flickr.com/1/222_b_m.jpg", + page="https://www.flickr.com/photos/x/222/"), + index_entry("C,_E3", "https://farm1.static.flickr.com/1/333_c_m.jpg"), + index_entry("D,_E4", "junk"), + ]).encode() + + def test_lists_missing_photos_with_pages_and_reasons(self): + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"x") + rb.writejson(os.path.join(self.dir, rb.FAILURES), + {"222": {"reason": "photo 222: HTTP 404 Not Found", "pages": ["B, E2"], + "when": "2026-09-13T10:00:00Z"}}) + rb.OPEN = FakeOpen(FakeResponse(self.index())) + self.assertEqual(rb.run(self.conf()), 0) + out = self.stdout.getvalue() + self.assertNotIn("111", out) + self.assertIn("222: photo 222: HTTP 404 Not Found (2026-09-13)", out) + self.assertIn(" B, E2 https://london.randomness.org.uk/wiki.cgi?B%2C_E2", out) + self.assertIn("333: not yet attempted", out) + self.assertIn(" C, E3 ", out) + self.assertIn("D,_E4: node_image is not a url", out) + self.assertIn("2 of 3 photos missing, 1 node with an unusable image", out) + self.assertEqual(len(rb.OPEN.requests), 1) # the index only + + def test_report_prints_even_when_quiet(self): + rb.OPEN = FakeOpen(FakeResponse(self.index())) + rb.run(self.conf(quiet=True)) + self.assertIn("3 of 3 photos missing", self.stdout.getvalue()) + + def test_report_needs_no_credentials(self): + rb.OPEN = FakeOpen(FakeResponse(b"[]")) + self.assertEqual(rb.run(self.conf()), 0) + self.assertIn("0 of 0 photos missing", self.stdout.getvalue()) + + +class RunRecordsFailures(TestCase): + def conf(self, **over): + conf = {"directory": self.dir, "flickr_auth": self.write("f.yml", "key: FK\n"), + "immich_auth": self.write("i.yml", "url: https://i/api\nkey: IK\n"), + "index_url": "http://t/index", "fetch": True, "sync": False, "report": False, + "dry_run": False, "limit": None, "pace": 0, "quiet": False, + "attempts": 2, "backoff": 1, "timeout": 5, "user_agent": "ua", "debug": False} + conf.update(over) + return conf + + def test_failed_run_writes_the_failures_file(self): + index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")] + rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode()), + FakeResponse({"stat": "fail", "code": 1, "message": "Photo not found"})) + self.assertEqual(rb.run(self.conf()), 1) + held = rb.loadfailures(self.dir) + self.assertIn("111", held) + self.assertIn("Photo not found", held["111"]["reason"]) + self.assertEqual(held["111"]["pages"], ["A, E1"]) + + def test_dry_run_writes_nothing(self): + index = [index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg")] + rb.OPEN = FakeOpen(FakeResponse(json.dumps(index).encode())) + rb.run(self.conf(dry_run=True)) + self.assertFalse(os.path.exists(os.path.join(self.dir, rb.FAILURES)))