Mirror the backup directory into an immich album

List what immich already holds by checksum, upload only what is missing,
then bring every matched asset up to date: the marked description block,
coordinates and capture date where absent, the RGL album, and licence and
photographer tags. Album membership and descriptions are read-compare-write;
uploaded assets are re-read from immich after the album and tag stage so a
duplicate answer or a late metadata job never sees a blank slate. Tag
assignments are re-sent each run and immich reports the duplicates.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

rgl-photo-backup

@@ -1129,5 +1129,154 @@ class Immich:
         return (tagged, already)
 
 
+def isoutc(epoch):
+    return datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+#
+# When the photo was taken, for immich's timeline. Flickr strips EXIF from
+# every size but the original, so without this most photos would file under
+# the upload date. Flickr's "taken" is local time with no zone; treat it as
+# UTC rather than guess.
+#
+def flickrtaken(side):
+    taken = (side.get("flickr") or {}).get("taken") or ""
+    try:
+        when = datetime.datetime.strptime(taken, "%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        return None
+    return when.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+def takenat(side, path):
+    return flickrtaken(side) or isoutc(os.stat(path).st_mtime)
+
+def coordinates(side):
+    for node in (side.get("rgl") or {}).get("nodes") or []:
+        try:
+            lat, lon = float(node.get("lat")), float(node.get("long"))
+        except (TypeError, ValueError):
+            continue
+        if lat or lon:
+            return (lat, lon)
+    return None
+
+#
+# What to write to an asset: only what is missing or out of date. Existing
+# coordinates and dates are never replaced; the description is appended to
+# or its marked block refreshed.
+#
+def planasset(side, asset):
+    fields = {}
+    text, action = mergedescription(asset.get("description"), describeblock(side))
+    if action != "same":
+        fields["description"] = text
+    if asset.get("latitude") is None and asset.get("longitude") is None:
+        where = coordinates(side)
+        if where:
+            fields["latitude"], fields["longitude"] = where
+    if not asset.get("taken"):
+        taken = flickrtaken(side)
+        if taken:
+            fields["dateTimeOriginal"] = taken
+    return fields
+
+def syncphase(conf, immich):
+    failures = Failures()
+    directory = conf["directory"]
+    dry = conf["dry_run"]
+
+    owner = immich.me()["id"]
+    held = immich.assets(owner, progress=lambda n: debug("immich: %d assets listed" % n))
+    say("immich: %d assets already held" % len(held))
+
+    hashes = Hashes(os.path.join(directory, HASHES))
+    matched = {}                      # asset id -> sidecar
+    fresh = {}                        # asset id -> sidecar, annotated at the end
+    tried = 0
+    for key, side in listsidecars(directory):
+        if conf["limit"] is not None and tried >= conf["limit"]:
+            break
+        try:
+            name = side.get("file")
+            if not name:
+                raise BackupError("%s: sidecar names no file" % key)
+            path = os.path.join(directory, name)
+            if not os.path.isfile(path):
+                continue
+            tried += 1
+            digest = hashes.of(path, name, os.stat(path))
+            asset = held.get(digest)
+            if asset is None:
+                if dry:
+                    say("%s: would upload %s" % (key, name))
+                    continue
+                assetid, status = immich.upload(path, digest, key, takenat(side, path),
+                                                isoutc(os.stat(path).st_mtime))
+                STATS["uploaded" if status == "created" else "existing"] += 1
+                say("%s: %s as %s" % (key, status, assetid))
+                # A second sidecar for a byte-identical file must reuse this
+                # asset, or immich would call it a duplicate and the two would
+                # take turns overwriting each other's description.
+                held[digest] = asset = {"id": assetid, "new": True}
+            else:
+                STATS["existing"] += 1
+            if asset.get("new"):
+                # immich extracts metadata after the upload returns, and a
+                # duplicate means an asset with a description of its own, so
+                # these are planned against what immich holds, further down.
+                fresh[asset["id"]] = side
+            else:
+                fields = planasset(side, asset)
+                if fields:
+                    if dry:
+                        say("%s: would set %s" % (key, ", ".join(sorted(fields))))
+                    else:
+                        immich.update(asset["id"], **fields)
+                        STATS["updated"] += 1
+            matched[asset["id"]] = side
+        except BackupError as e:
+            failures.add(key, sidenodes(side), e)
+        except OSError as e:
+            failures.add(key, sidenodes(side), "%s: %s" % (type(e).__name__, e))
+    if not dry:
+        hashes.save()
+
+    try:
+        albums = immich.albums(owner)
+        if dry:
+            say("album %s: %s, %d asset(s) would be filed" % (ALBUM, "exists" if ALBUM in albums else "would be created", len(matched)))
+            return failures
+        albumid = albums.get(ALBUM) or immich.createalbum(ALBUM, "Photos used on the Randomness Guide to London")
+        inalbum = immich.albumassets(albumid)
+        missing = [a for a in matched if a not in inalbum]
+        if missing:
+            say("album %s: filing %d asset(s)" % (ALBUM, immich.addtoalbum(albumid, missing)))
+
+        wanted = {}
+        for assetid, side in matched.items():
+            for tag in tagsfor(side):
+                wanted.setdefault(tag, []).append(assetid)
+        if wanted:
+            ids = immich.tagids(sorted(wanted))
+            for tag in sorted(wanted):
+                if tag not in ids:
+                    failures.add(tag, [], "immich returned no id for this tag")
+                    continue
+                tagged, already = immich.tagassets(ids[tag], wanted[tag])
+                if tagged:
+                    say("tag %s: %d tagged, %d already" % (tag, tagged, already))
+    except BackupError as e:
+        failures.add("album/tags", [], e)
+
+    for assetid, side in fresh.items():
+        try:
+            fields = planasset(side, immich.asset(assetid))
+            if fields:
+                immich.update(assetid, **fields)
+                STATS["updated"] += 1
+        except BackupError as e:
+            failures.add(side.get("key", assetid), sidenodes(side), e)
+    return failures
+
+
 if __name__ == "__main__":
     sys.exit(0)

test_rgl_photo_backup.py

@@ -1114,3 +1114,253 @@ class Hashing(TestCase):
         os.utime(path, (1, 1))
         self.assertNotEqual(again.of(path, "f", os.stat(path)), first)
         self.assertTrue(again.dirty)
+
+
+class PlanAsset(TestCase):
+    def asset(self, **over):
+        a = {"id": "a1", "name": "111.jpg", "description": "", "latitude": None,
+             "longitude": None, "taken": ""}
+        a.update(over)
+        return a
+
+    def test_fresh_asset_gets_everything(self):
+        fields = rb.planasset(sidecar(), self.asset())
+        self.assertEqual(fields["description"], rb.describeblock(sidecar()))
+        self.assertEqual((fields["latitude"], fields["longitude"]), (51.5, -0.1))
+        self.assertEqual(fields["dateTimeOriginal"], "2009-07-22T14:03:11Z")
+
+    def test_existing_values_are_kept(self):
+        a = self.asset(description="Mine\n\n" + rb.describeblock(sidecar()),
+                       latitude=1.0, longitude=2.0, taken="2001-01-01T00:00:00.000Z")
+        self.assertEqual(rb.planasset(sidecar(), a), {})
+
+    def test_hand_written_description_is_appended_to(self):
+        fields = rb.planasset(sidecar(), self.asset(description="Mine", latitude=1.0, longitude=2.0, taken="t"))
+        self.assertEqual(list(fields), ["description"])
+        self.assertTrue(fields["description"].startswith("Mine\n\n"))
+
+    def test_bad_coordinates_are_skipped(self):
+        side = sidecar(nodes=[{"param": "A", "name": "A", "url": "u", "lat": "", "long": ""}])
+        fields = rb.planasset(side, self.asset())
+        self.assertNotIn("latitude", fields)
+
+    def test_flickrtaken(self):
+        self.assertEqual(rb.flickrtaken(sidecar()), "2009-07-22T14:03:11Z")
+        self.assertIsNone(rb.flickrtaken(sidecar(flickr=False)))
+        side = sidecar()
+        side["flickr"]["taken"] = "garbage"
+        self.assertIsNone(rb.flickrtaken(side))
+
+    def test_takenat(self):
+        path = self.write("111.jpg", b"x")
+        os.utime(path, (1000000000, 1000000000))
+        self.assertEqual(rb.takenat(sidecar(), path), "2009-07-22T14:03:11Z")
+        self.assertEqual(rb.takenat(sidecar(flickr=False), path), "2001-09-09T01:46:40Z")
+        side = sidecar()
+        side["flickr"]["taken"] = "garbage"
+        self.assertEqual(rb.takenat(side, path), "2001-09-09T01:46:40Z")
+
+
+class SyncPhase(TestCase):
+    def setUp(self):
+        super().setUp()
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"NEW")
+        rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg", copyright="Q"))
+        self.write("222.jpg", b"OLD")
+        self.old = rb.checksum(os.path.join(self.dir, "222.jpg"))
+
+    def conf(self, **over):
+        conf = {"directory": self.dir, "dry_run": False, "quiet": False, "limit": None}
+        conf.update(over)
+        return conf
+
+    def responses(self):
+        return [
+            FakeResponse({"id": "u1"}),                                         # me
+            FakeResponse({"assets": {"items": [
+                {"id": "a2", "ownerId": "u1", "checksum": self.old,
+                 "exifInfo": {"description": "Mine", "latitude": 1, "longitude": 2,
+                              "dateTimeOriginal": "t"}}], "nextPage": None}}),   # assets
+            FakeResponse({"id": "a1", "status": "created"}, status=201),        # upload 111
+            FakeResponse(b""),                                                  # update a2 (append)
+            FakeResponse([]),                                                   # albums
+            FakeResponse({"id": "al1"}),                                        # create album
+            FakeResponse({"assets": []}),                                       # album assets
+            FakeResponse([{"success": True}, {"success": True}]),               # add to album
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"},
+                          {"value": "rgl/photographer/Q", "id": "t4"}]),        # tags
+            FakeResponse([{"success": True}, {"success": True}]),               # t1
+            FakeResponse([{"success": True}, {"success": True}]),               # t2
+            FakeResponse([{"success": True}]),                                  # t3
+            FakeResponse([{"success": True}]),                                  # t4
+            FakeResponse({"id": "a1", "originalFileName": "111.jpg",
+                          "exifInfo": {}}),                                     # re-read a1
+            FakeResponse(b""),                                                  # update a1
+        ]
+
+    def test_uploads_new_reuses_existing_and_annotates(self):
+        rb.OPEN = FakeOpen(*self.responses())
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        self.assertEqual(rb.STATS["uploaded"], 1)
+        self.assertEqual(rb.STATS["existing"], 1)
+        reqs = rb.OPEN.requests
+        self.assertTrue(reqs[2].full_url.endswith("/assets"))
+        upd2 = json.loads(reqs[3].data)
+        self.assertEqual(list(upd2), ["description"])
+        self.assertTrue(upd2["description"].startswith("Mine\n\n"))
+        self.assertEqual(json.loads(reqs[5].data)["albumName"], "RGL")
+        self.assertEqual(sorted(json.loads(reqs[7].data)["ids"]), ["a1", "a2"])
+        # the uploaded asset is annotated last, against what immich now holds
+        self.assertEqual((reqs[13].get_method(), reqs[13].full_url),
+                         ("GET", "https://i/api/assets/a1"))
+        upd1 = json.loads(reqs[14].data)
+        self.assertIn("latitude", upd1)
+        self.assertEqual(upd1["description"], rb.describeblock(sidecar()))
+        self.assertTrue(os.path.exists(os.path.join(self.dir, rb.HASHES)))
+
+    def test_duplicate_upload_appends_to_what_immich_already_holds(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "duplicate"}),                  # upload 111
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": []}),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a1", "originalFileName": "mine.jpg",
+                          "exifInfo": {"description": "My own words",
+                                       "latitude": 1, "longitude": 2,
+                                       "dateTimeOriginal": "2001-01-01T00:00:00.000Z"}}),
+            FakeResponse(b""))
+        failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        self.assertEqual(rb.STATS["uploaded"], 0)
+        self.assertEqual(rb.STATS["existing"], 1)
+        last = rb.OPEN.requests[-1]
+        self.assertEqual((last.get_method(), last.full_url),
+                         ("PUT", "https://i/api/assets/a1"))
+        body = json.loads(last.data)
+        self.assertEqual(list(body), ["description"])
+        self.assertTrue(body["description"].startswith("My own words\n\n"))
+
+    def test_a_repeated_checksum_is_uploaded_once(self):
+        self.write("222.jpg", b"NEW")               # byte-identical to 111.jpg
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "created"}, status=201),        # upload 111
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": [{"id": "a1"}]}),
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/Q", "id": "t3"}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a1", "exifInfo": {}}),                         # re-read a1
+            FakeResponse(b""))
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        uploads = [r for r in rb.OPEN.requests if r.full_url.endswith("/api/assets")]
+        self.assertEqual(len(uploads), 1)
+        self.assertEqual(rb.STATS["uploaded"], 1)
+        self.assertEqual(rb.STATS["existing"], 1)
+
+    def test_second_run_is_quiet(self):
+        block1 = rb.describeblock(sidecar())
+        block2 = rb.describeblock(sidecar(key="222", file="222.jpg", copyright="Q"))
+        new = rb.checksum(os.path.join(self.dir, "111.jpg"))
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [
+                {"id": "a1", "ownerId": "u1", "checksum": new,
+                 "exifInfo": {"description": block1, "latitude": 51.5, "longitude": -0.1, "dateTimeOriginal": "t"}},
+                {"id": "a2", "ownerId": "u1", "checksum": self.old,
+                 "exifInfo": {"description": "Mine\n\n" + block2, "latitude": 1, "longitude": 2, "dateTimeOriginal": "t"}}],
+                "nextPage": None}}),
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": [{"id": "a1"}, {"id": "a2"}]}),
+            FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"}, {"value": "rgl/photographer/Q", "id": "t4"}]),
+            FakeResponse([{"error": "duplicate"}, {"error": "duplicate"}]),
+            FakeResponse([{"error": "duplicate"}, {"error": "duplicate"}]),
+            FakeResponse([{"error": "duplicate"}]),
+            FakeResponse([{"error": "duplicate"}]))
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        self.assertEqual(rb.STATS["uploaded"], 0)
+        self.assertEqual(rb.STATS["updated"], 0)
+
+    def test_dry_run_only_reads(self):
+        rb.OPEN = FakeOpen(FakeResponse({"id": "u1"}),
+                           FakeResponse({"assets": {"items": [], "nextPage": None}}),
+                           FakeResponse([]))
+        rb.syncphase(self.conf(dry_run=True), rb.Immich("https://i/api", "k"))
+        self.assertEqual([r.get_method() for r in rb.OPEN.requests], ["GET", "POST", "GET"])
+        self.assertIn("would upload", self.stdout.getvalue())
+        self.assertFalse(os.path.exists(os.path.join(self.dir, rb.HASHES)))
+
+    def test_dry_run_limit_counts_what_it_looked_at(self):
+        rb.OPEN = FakeOpen(FakeResponse({"id": "u1"}),
+                           FakeResponse({"assets": {"items": [], "nextPage": None}}),
+                           FakeResponse([]))
+        rb.syncphase(self.conf(dry_run=True, limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(self.stdout.getvalue().count("would upload"), 1)
+
+    def test_unsupported_file_is_a_failure_not_a_stop(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            http_error(400, b'{"message":"unsupported file type"}'),   # upload 111
+            FakeResponse({"id": "a2", "status": "created"}, status=201),  # upload 222
+            FakeResponse([]), FakeResponse({"id": "al1"}), FakeResponse({"assets": []}),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/Q", "id": "t4"}]),
+            FakeResponse([{"success": True}]), FakeResponse([{"success": True}]), FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a2", "exifInfo": {}}),                   # re-read a2
+            FakeResponse(b""))                                            # update a2
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 1)
+        self.assertIn("unsupported", failures.items[0][2])
+        self.assertEqual(rb.STATS["uploaded"], 1)
+
+    def test_malformed_sidecar_is_a_failure_not_a_stop(self):
+        room = os.path.join(self.dir, "other")
+        os.mkdir(room)
+        rb.writejson(os.path.join(room, "333.json"), {"key": "333"})
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse([]), FakeResponse({"id": "al1"}), FakeResponse({"assets": []}))
+        failures = rb.syncphase(self.conf(directory=room), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 1)
+        self.assertIn("no file", failures.items[0][2])
+
+    def test_bad_key_is_fatal_before_writes(self):
+        rb.OPEN = FakeOpen(http_error(401))
+        with self.assertRaises(rb.BackupError):
+            rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+
+    def test_album_stage_failure_is_recorded(self):
+        rb.ATTEMPTS = 1
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "created"}, status=201),   # upload 111
+            http_error(500),                                              # albums
+            FakeResponse({"id": "a1", "exifInfo": {}}),                   # re-read a1
+            FakeResponse(b""))                                            # update a1
+        failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 1)
+        reason = failures.items[0][2]
+        self.assertTrue("albums" in reason or "HTTP 500" in reason)