Mirror the backup directory into an immich album

List what immich already holds by checksum, upload only what is missing,
then bring every matched asset up to date: the marked description block,
coordinates and capture date where absent, the RGL album, and licence and
photographer tags. Album membership and descriptions are read-compare-write;
uploaded assets are re-read from immich after the album and tag stage so a
duplicate answer or a late metadata job never sees a blank slate. Tag
assignments are re-sent each run and immich reports the duplicates.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

test_rgl_photo_backup.py

@@ -1114,3 +1114,253 @@ class Hashing(TestCase):
         os.utime(path, (1, 1))
         self.assertNotEqual(again.of(path, "f", os.stat(path)), first)
         self.assertTrue(again.dirty)
+
+
+class PlanAsset(TestCase):
+    def asset(self, **over):
+        a = {"id": "a1", "name": "111.jpg", "description": "", "latitude": None,
+             "longitude": None, "taken": ""}
+        a.update(over)
+        return a
+
+    def test_fresh_asset_gets_everything(self):
+        fields = rb.planasset(sidecar(), self.asset())
+        self.assertEqual(fields["description"], rb.describeblock(sidecar()))
+        self.assertEqual((fields["latitude"], fields["longitude"]), (51.5, -0.1))
+        self.assertEqual(fields["dateTimeOriginal"], "2009-07-22T14:03:11Z")
+
+    def test_existing_values_are_kept(self):
+        a = self.asset(description="Mine\n\n" + rb.describeblock(sidecar()),
+                       latitude=1.0, longitude=2.0, taken="2001-01-01T00:00:00.000Z")
+        self.assertEqual(rb.planasset(sidecar(), a), {})
+
+    def test_hand_written_description_is_appended_to(self):
+        fields = rb.planasset(sidecar(), self.asset(description="Mine", latitude=1.0, longitude=2.0, taken="t"))
+        self.assertEqual(list(fields), ["description"])
+        self.assertTrue(fields["description"].startswith("Mine\n\n"))
+
+    def test_bad_coordinates_are_skipped(self):
+        side = sidecar(nodes=[{"param": "A", "name": "A", "url": "u", "lat": "", "long": ""}])
+        fields = rb.planasset(side, self.asset())
+        self.assertNotIn("latitude", fields)
+
+    def test_flickrtaken(self):
+        self.assertEqual(rb.flickrtaken(sidecar()), "2009-07-22T14:03:11Z")
+        self.assertIsNone(rb.flickrtaken(sidecar(flickr=False)))
+        side = sidecar()
+        side["flickr"]["taken"] = "garbage"
+        self.assertIsNone(rb.flickrtaken(side))
+
+    def test_takenat(self):
+        path = self.write("111.jpg", b"x")
+        os.utime(path, (1000000000, 1000000000))
+        self.assertEqual(rb.takenat(sidecar(), path), "2009-07-22T14:03:11Z")
+        self.assertEqual(rb.takenat(sidecar(flickr=False), path), "2001-09-09T01:46:40Z")
+        side = sidecar()
+        side["flickr"]["taken"] = "garbage"
+        self.assertEqual(rb.takenat(side, path), "2001-09-09T01:46:40Z")
+
+
+class SyncPhase(TestCase):
+    def setUp(self):
+        super().setUp()
+        rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar())
+        self.write("111.jpg", b"NEW")
+        rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg", copyright="Q"))
+        self.write("222.jpg", b"OLD")
+        self.old = rb.checksum(os.path.join(self.dir, "222.jpg"))
+
+    def conf(self, **over):
+        conf = {"directory": self.dir, "dry_run": False, "quiet": False, "limit": None}
+        conf.update(over)
+        return conf
+
+    def responses(self):
+        return [
+            FakeResponse({"id": "u1"}),                                         # me
+            FakeResponse({"assets": {"items": [
+                {"id": "a2", "ownerId": "u1", "checksum": self.old,
+                 "exifInfo": {"description": "Mine", "latitude": 1, "longitude": 2,
+                              "dateTimeOriginal": "t"}}], "nextPage": None}}),   # assets
+            FakeResponse({"id": "a1", "status": "created"}, status=201),        # upload 111
+            FakeResponse(b""),                                                  # update a2 (append)
+            FakeResponse([]),                                                   # albums
+            FakeResponse({"id": "al1"}),                                        # create album
+            FakeResponse({"assets": []}),                                       # album assets
+            FakeResponse([{"success": True}, {"success": True}]),               # add to album
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"},
+                          {"value": "rgl/photographer/Q", "id": "t4"}]),        # tags
+            FakeResponse([{"success": True}, {"success": True}]),               # t1
+            FakeResponse([{"success": True}, {"success": True}]),               # t2
+            FakeResponse([{"success": True}]),                                  # t3
+            FakeResponse([{"success": True}]),                                  # t4
+            FakeResponse({"id": "a1", "originalFileName": "111.jpg",
+                          "exifInfo": {}}),                                     # re-read a1
+            FakeResponse(b""),                                                  # update a1
+        ]
+
+    def test_uploads_new_reuses_existing_and_annotates(self):
+        rb.OPEN = FakeOpen(*self.responses())
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        self.assertEqual(rb.STATS["uploaded"], 1)
+        self.assertEqual(rb.STATS["existing"], 1)
+        reqs = rb.OPEN.requests
+        self.assertTrue(reqs[2].full_url.endswith("/assets"))
+        upd2 = json.loads(reqs[3].data)
+        self.assertEqual(list(upd2), ["description"])
+        self.assertTrue(upd2["description"].startswith("Mine\n\n"))
+        self.assertEqual(json.loads(reqs[5].data)["albumName"], "RGL")
+        self.assertEqual(sorted(json.loads(reqs[7].data)["ids"]), ["a1", "a2"])
+        # the uploaded asset is annotated last, against what immich now holds
+        self.assertEqual((reqs[13].get_method(), reqs[13].full_url),
+                         ("GET", "https://i/api/assets/a1"))
+        upd1 = json.loads(reqs[14].data)
+        self.assertIn("latitude", upd1)
+        self.assertEqual(upd1["description"], rb.describeblock(sidecar()))
+        self.assertTrue(os.path.exists(os.path.join(self.dir, rb.HASHES)))
+
+    def test_duplicate_upload_appends_to_what_immich_already_holds(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "duplicate"}),                  # upload 111
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": []}),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a1", "originalFileName": "mine.jpg",
+                          "exifInfo": {"description": "My own words",
+                                       "latitude": 1, "longitude": 2,
+                                       "dateTimeOriginal": "2001-01-01T00:00:00.000Z"}}),
+            FakeResponse(b""))
+        failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        self.assertEqual(rb.STATS["uploaded"], 0)
+        self.assertEqual(rb.STATS["existing"], 1)
+        last = rb.OPEN.requests[-1]
+        self.assertEqual((last.get_method(), last.full_url),
+                         ("PUT", "https://i/api/assets/a1"))
+        body = json.loads(last.data)
+        self.assertEqual(list(body), ["description"])
+        self.assertTrue(body["description"].startswith("My own words\n\n"))
+
+    def test_a_repeated_checksum_is_uploaded_once(self):
+        self.write("222.jpg", b"NEW")               # byte-identical to 111.jpg
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "created"}, status=201),        # upload 111
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": [{"id": "a1"}]}),
+            FakeResponse([{"value": "rgl", "id": "t1"},
+                          {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/Q", "id": "t3"}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a1", "exifInfo": {}}),                         # re-read a1
+            FakeResponse(b""))
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        uploads = [r for r in rb.OPEN.requests if r.full_url.endswith("/api/assets")]
+        self.assertEqual(len(uploads), 1)
+        self.assertEqual(rb.STATS["uploaded"], 1)
+        self.assertEqual(rb.STATS["existing"], 1)
+
+    def test_second_run_is_quiet(self):
+        block1 = rb.describeblock(sidecar())
+        block2 = rb.describeblock(sidecar(key="222", file="222.jpg", copyright="Q"))
+        new = rb.checksum(os.path.join(self.dir, "111.jpg"))
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [
+                {"id": "a1", "ownerId": "u1", "checksum": new,
+                 "exifInfo": {"description": block1, "latitude": 51.5, "longitude": -0.1, "dateTimeOriginal": "t"}},
+                {"id": "a2", "ownerId": "u1", "checksum": self.old,
+                 "exifInfo": {"description": "Mine\n\n" + block2, "latitude": 1, "longitude": 2, "dateTimeOriginal": "t"}}],
+                "nextPage": None}}),
+            FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]),
+            FakeResponse({"assets": [{"id": "a1"}, {"id": "a2"}]}),
+            FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/P", "id": "t3"}, {"value": "rgl/photographer/Q", "id": "t4"}]),
+            FakeResponse([{"error": "duplicate"}, {"error": "duplicate"}]),
+            FakeResponse([{"error": "duplicate"}, {"error": "duplicate"}]),
+            FakeResponse([{"error": "duplicate"}]),
+            FakeResponse([{"error": "duplicate"}]))
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 0)
+        self.assertEqual(rb.STATS["uploaded"], 0)
+        self.assertEqual(rb.STATS["updated"], 0)
+
+    def test_dry_run_only_reads(self):
+        rb.OPEN = FakeOpen(FakeResponse({"id": "u1"}),
+                           FakeResponse({"assets": {"items": [], "nextPage": None}}),
+                           FakeResponse([]))
+        rb.syncphase(self.conf(dry_run=True), rb.Immich("https://i/api", "k"))
+        self.assertEqual([r.get_method() for r in rb.OPEN.requests], ["GET", "POST", "GET"])
+        self.assertIn("would upload", self.stdout.getvalue())
+        self.assertFalse(os.path.exists(os.path.join(self.dir, rb.HASHES)))
+
+    def test_dry_run_limit_counts_what_it_looked_at(self):
+        rb.OPEN = FakeOpen(FakeResponse({"id": "u1"}),
+                           FakeResponse({"assets": {"items": [], "nextPage": None}}),
+                           FakeResponse([]))
+        rb.syncphase(self.conf(dry_run=True, limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(self.stdout.getvalue().count("would upload"), 1)
+
+    def test_unsupported_file_is_a_failure_not_a_stop(self):
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            http_error(400, b'{"message":"unsupported file type"}'),   # upload 111
+            FakeResponse({"id": "a2", "status": "created"}, status=201),  # upload 222
+            FakeResponse([]), FakeResponse({"id": "al1"}), FakeResponse({"assets": []}),
+            FakeResponse([{"success": True}]),
+            FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"},
+                          {"value": "rgl/photographer/Q", "id": "t4"}]),
+            FakeResponse([{"success": True}]), FakeResponse([{"success": True}]), FakeResponse([{"success": True}]),
+            FakeResponse({"id": "a2", "exifInfo": {}}),                   # re-read a2
+            FakeResponse(b""))                                            # update a2
+        failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 1)
+        self.assertIn("unsupported", failures.items[0][2])
+        self.assertEqual(rb.STATS["uploaded"], 1)
+
+    def test_malformed_sidecar_is_a_failure_not_a_stop(self):
+        room = os.path.join(self.dir, "other")
+        os.mkdir(room)
+        rb.writejson(os.path.join(room, "333.json"), {"key": "333"})
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse([]), FakeResponse({"id": "al1"}), FakeResponse({"assets": []}))
+        failures = rb.syncphase(self.conf(directory=room), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 1)
+        self.assertIn("no file", failures.items[0][2])
+
+    def test_bad_key_is_fatal_before_writes(self):
+        rb.OPEN = FakeOpen(http_error(401))
+        with self.assertRaises(rb.BackupError):
+            rb.syncphase(self.conf(), rb.Immich("https://i/api", "k"))
+
+    def test_album_stage_failure_is_recorded(self):
+        rb.ATTEMPTS = 1
+        rb.OPEN = FakeOpen(
+            FakeResponse({"id": "u1"}),
+            FakeResponse({"assets": {"items": [], "nextPage": None}}),
+            FakeResponse({"id": "a1", "status": "created"}, status=201),   # upload 111
+            http_error(500),                                              # albums
+            FakeResponse({"id": "a1", "exifInfo": {}}),                   # re-read a1
+            FakeResponse(b""))                                            # update a1
+        failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k"))
+        self.assertEqual(len(failures), 1)
+        reason = failures.items[0][2]
+        self.assertTrue("albums" in reason or "HTTP 500" in reason)