From f7884b943ee923556afa6006726bc9562ce9cf08 Mon Sep 17 00:00:00 2001 From: bob walker Date: Thu, 10 Sep 2026 12:07:44 +0100 Subject: [PATCH] Store each photo's metadata in a json sidecar beside it The sidecar is what makes the directory a backup on its own and what the immich phase reads, so it is written atomically and a photo only counts as done when the sidecar, its file and no .part remain. Co-Authored-By: Claude Fable 5.1 --- rgl-photo-backup | 55 ++++++++++++++++++++++++++++++++++++ test_rgl_photo_backup.py | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/rgl-photo-backup b/rgl-photo-backup index 408a4aa..68d9f55 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -600,5 +600,60 @@ def extensionfor(source, contenttype=""): return CONTENT_EXTENSIONS.get(kind, ".jpg") +def utcnow(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + +def sidecarpath(directory, key): + return os.path.join(directory, key + ".json") + +def writejson(path, data): + partial = path + ".part" + with open(partial, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=2, ensure_ascii=False, sort_keys=True) + fh.write("\n") + os.replace(partial, path) + +def readsidecar(path): + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + return None + return data if isinstance(data, dict) and data.get("key") else None + +def mediapath(directory, key): + try: + names = os.listdir(directory) + except OSError: + return None + for name in sorted(names): + stem, ext = os.path.splitext(name) + if stem == key and ext not in (".json", ".part"): + return os.path.join(directory, name) + return None + +def iscomplete(directory, key): + if os.path.exists(os.path.join(directory, key + ".part")): + return False + side = readsidecar(sidecarpath(directory, key)) + if not side or not side.get("file"): + return False + return os.path.isfile(os.path.join(directory, side["file"])) + +def listsidecars(directory): + found = [] + try: + names = sorted(os.listdir(directory)) + except OSError as e: + raise BackupError("%s: cannot be listed: %s" % (directory, e.strerror or e)) + for name in names: + if not name.endswith(".json") or name == HASHES: + continue + side = readsidecar(os.path.join(directory, name)) + if side: + found.append((side["key"], side)) + return found + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index ffbb06a..96850b1 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -541,3 +541,63 @@ class Flickr(TestCase): self.assertEqual(rb.extensionfor("https://x/1_o.jpeg", ""), ".jpg") self.assertEqual(rb.extensionfor("https://x/1_o", "image/gif"), ".gif") self.assertEqual(rb.extensionfor("https://x/1_o.weird", "text/html"), ".jpg") + + +def sidecar(key="111", file="111.jpg", nodes=None, copyright="P", licence="https://creativecommons.org/licenses/by-sa/2.0/", flickr=True): + data = {"key": key, "file": file, "source": "flickr" if flickr else "url", + "rgl": {"copyright": copyright, "licence": licence, + "nodes": nodes or [{"param": "A,_E1", "name": "A, E1", + "url": "https://london.randomness.org.uk/wiki.cgi?A%2C_E1", + "lat": "51.5", "long": "-0.1"}]}, + "fetched": "2026-09-09T21:14:00Z"} + if flickr: + data["flickr"] = {"id": key, "owner": "12345678@N00", "owner_name": "handle", + "owner_realname": "Real Name", "title": "A title", + "taken": "2009-07-22 14:03:11", "posted": "2009-07-23T10:40:00Z", + "licence_id": 5, "page": "https://www.flickr.com/photos/12345678@N00/%s/" % key, + "size_label": "Original", "width": 800, "height": 600} + else: + data["page"] = "https://www.geograph.org.uk/photos/1.jpg" + return data + + +class Sidecars(TestCase): + def test_round_trip_is_atomic(self): + path = rb.sidecarpath(self.dir, "111") + self.assertEqual(path, os.path.join(self.dir, "111.json")) + rb.writejson(path, sidecar()) + self.assertEqual(rb.readsidecar(path)["key"], "111") + self.assertFalse(os.path.exists(path + ".part")) + + def test_unreadable_sidecar_is_none(self): + self.assertIsNone(rb.readsidecar(os.path.join(self.dir, "none.json"))) + self.write("bad.json", "{not json") + self.assertIsNone(rb.readsidecar(os.path.join(self.dir, "bad.json"))) + + def test_mediapath_finds_the_photo_not_the_sidecar(self): + self.write("111.json", "{}") + self.write("111.part", b"x") + self.assertIsNone(rb.mediapath(self.dir, "111")) + self.write("111.png", b"x") + self.assertEqual(rb.mediapath(self.dir, "111"), os.path.join(self.dir, "111.png")) + + def test_iscomplete(self): + self.assertFalse(rb.iscomplete(self.dir, "111")) + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.assertFalse(rb.iscomplete(self.dir, "111")) # no file yet + self.write("111.jpg", b"x") + self.assertTrue(rb.iscomplete(self.dir, "111")) + self.write("111.part", b"x") + self.assertFalse(rb.iscomplete(self.dir, "111")) # interrupted refetch + def test_listsidecars_unreadable_directory_is_an_error(self): + with self.assertRaises(rb.BackupError): + rb.listsidecars(os.path.join(self.dir, "nowhere")) + + def test_listsidecars_sorted_and_skips_hash_cache(self): + rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg")) + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + rb.writejson(os.path.join(self.dir, rb.HASHES), {}) + self.assertEqual([k for k, _ in rb.listsidecars(self.dir)], ["111", "222"]) + + def test_utcnow_shape(self): + self.assertRegex(rb.utcnow(), r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$")