diff --git a/rgl-photo-backup b/rgl-photo-backup index bcdc77b..c2cbea4 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -1129,5 +1129,154 @@ class Immich: return (tagged, already) +def isoutc(epoch): + return datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + +# +# When the photo was taken, for immich's timeline. Flickr strips EXIF from +# every size but the original, so without this most photos would file under +# the upload date. Flickr's "taken" is local time with no zone; treat it as +# UTC rather than guess. +# +def flickrtaken(side): + taken = (side.get("flickr") or {}).get("taken") or "" + try: + when = datetime.datetime.strptime(taken, "%Y-%m-%d %H:%M:%S") + except ValueError: + return None + return when.strftime("%Y-%m-%dT%H:%M:%SZ") + +def takenat(side, path): + return flickrtaken(side) or isoutc(os.stat(path).st_mtime) + +def coordinates(side): + for node in (side.get("rgl") or {}).get("nodes") or []: + try: + lat, lon = float(node.get("lat")), float(node.get("long")) + except (TypeError, ValueError): + continue + if lat or lon: + return (lat, lon) + return None + +# +# What to write to an asset: only what is missing or out of date. Existing +# coordinates and dates are never replaced; the description is appended to +# or its marked block refreshed. +# +def planasset(side, asset): + fields = {} + text, action = mergedescription(asset.get("description"), describeblock(side)) + if action != "same": + fields["description"] = text + if asset.get("latitude") is None and asset.get("longitude") is None: + where = coordinates(side) + if where: + fields["latitude"], fields["longitude"] = where + if not asset.get("taken"): + taken = flickrtaken(side) + if taken: + fields["dateTimeOriginal"] = taken + return fields + +def syncphase(conf, immich): + failures = Failures() + directory = conf["directory"] + dry = conf["dry_run"] + + owner = immich.me()["id"] + held = immich.assets(owner, progress=lambda n: debug("immich: %d assets listed" % n)) + say("immich: %d assets already held" % len(held)) + + hashes = Hashes(os.path.join(directory, HASHES)) + matched = {} # asset id -> sidecar + fresh = {} # asset id -> sidecar, annotated at the end + tried = 0 + for key, side in listsidecars(directory): + if conf["limit"] is not None and tried >= conf["limit"]: + break + try: + name = side.get("file") + if not name: + raise BackupError("%s: sidecar names no file" % key) + path = os.path.join(directory, name) + if not os.path.isfile(path): + continue + tried += 1 + digest = hashes.of(path, name, os.stat(path)) + asset = held.get(digest) + if asset is None: + if dry: + say("%s: would upload %s" % (key, name)) + continue + assetid, status = immich.upload(path, digest, key, takenat(side, path), + isoutc(os.stat(path).st_mtime)) + STATS["uploaded" if status == "created" else "existing"] += 1 + say("%s: %s as %s" % (key, status, assetid)) + # A second sidecar for a byte-identical file must reuse this + # asset, or immich would call it a duplicate and the two would + # take turns overwriting each other's description. + held[digest] = asset = {"id": assetid, "new": True} + else: + STATS["existing"] += 1 + if asset.get("new"): + # immich extracts metadata after the upload returns, and a + # duplicate means an asset with a description of its own, so + # these are planned against what immich holds, further down. + fresh[asset["id"]] = side + else: + fields = planasset(side, asset) + if fields: + if dry: + say("%s: would set %s" % (key, ", ".join(sorted(fields)))) + else: + immich.update(asset["id"], **fields) + STATS["updated"] += 1 + matched[asset["id"]] = side + except BackupError as e: + failures.add(key, sidenodes(side), e) + except OSError as e: + failures.add(key, sidenodes(side), "%s: %s" % (type(e).__name__, e)) + if not dry: + hashes.save() + + try: + albums = immich.albums(owner) + if dry: + say("album %s: %s, %d asset(s) would be filed" % (ALBUM, "exists" if ALBUM in albums else "would be created", len(matched))) + return failures + albumid = albums.get(ALBUM) or immich.createalbum(ALBUM, "Photos used on the Randomness Guide to London") + inalbum = immich.albumassets(albumid) + missing = [a for a in matched if a not in inalbum] + if missing: + say("album %s: filing %d asset(s)" % (ALBUM, immich.addtoalbum(albumid, missing))) + + wanted = {} + for assetid, side in matched.items(): + for tag in tagsfor(side): + wanted.setdefault(tag, []).append(assetid) + if wanted: + ids = immich.tagids(sorted(wanted)) + for tag in sorted(wanted): + if tag not in ids: + failures.add(tag, [], "immich returned no id for this tag") + continue + tagged, already = immich.tagassets(ids[tag], wanted[tag]) + if tagged: + say("tag %s: %d tagged, %d already" % (tag, tagged, already)) + except BackupError as e: + failures.add("album/tags", [], e) + + for assetid, side in fresh.items(): + try: + fields = planasset(side, immich.asset(assetid)) + if fields: + immich.update(assetid, **fields) + STATS["updated"] += 1 + except BackupError as e: + failures.add(side.get("key", assetid), sidenodes(side), e) + return failures + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index 9da690b..eb3f676 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -1114,3 +1114,253 @@ class Hashing(TestCase): os.utime(path, (1, 1)) self.assertNotEqual(again.of(path, "f", os.stat(path)), first) self.assertTrue(again.dirty) + + +class PlanAsset(TestCase): + def asset(self, **over): + a = {"id": "a1", "name": "111.jpg", "description": "", "latitude": None, + "longitude": None, "taken": ""} + a.update(over) + return a + + def test_fresh_asset_gets_everything(self): + fields = rb.planasset(sidecar(), self.asset()) + self.assertEqual(fields["description"], rb.describeblock(sidecar())) + self.assertEqual((fields["latitude"], fields["longitude"]), (51.5, -0.1)) + self.assertEqual(fields["dateTimeOriginal"], "2009-07-22T14:03:11Z") + + def test_existing_values_are_kept(self): + a = self.asset(description="Mine\n\n" + rb.describeblock(sidecar()), + latitude=1.0, longitude=2.0, taken="2001-01-01T00:00:00.000Z") + self.assertEqual(rb.planasset(sidecar(), a), {}) + + def test_hand_written_description_is_appended_to(self): + fields = rb.planasset(sidecar(), self.asset(description="Mine", latitude=1.0, longitude=2.0, taken="t")) + self.assertEqual(list(fields), ["description"]) + self.assertTrue(fields["description"].startswith("Mine\n\n")) + + def test_bad_coordinates_are_skipped(self): + side = sidecar(nodes=[{"param": "A", "name": "A", "url": "u", "lat": "", "long": ""}]) + fields = rb.planasset(side, self.asset()) + self.assertNotIn("latitude", fields) + + def test_flickrtaken(self): + self.assertEqual(rb.flickrtaken(sidecar()), "2009-07-22T14:03:11Z") + self.assertIsNone(rb.flickrtaken(sidecar(flickr=False))) + side = sidecar() + side["flickr"]["taken"] = "garbage" + self.assertIsNone(rb.flickrtaken(side)) + + def test_takenat(self): + path = self.write("111.jpg", b"x") + os.utime(path, (1000000000, 1000000000)) + self.assertEqual(rb.takenat(sidecar(), path), "2009-07-22T14:03:11Z") + self.assertEqual(rb.takenat(sidecar(flickr=False), path), "2001-09-09T01:46:40Z") + side = sidecar() + side["flickr"]["taken"] = "garbage" + self.assertEqual(rb.takenat(side, path), "2001-09-09T01:46:40Z") + + +class SyncPhase(TestCase): + def setUp(self): + super().setUp() + rb.writejson(rb.sidecarpath(self.dir, "111"), sidecar()) + self.write("111.jpg", b"NEW") + rb.writejson(rb.sidecarpath(self.dir, "222"), sidecar(key="222", file="222.jpg", copyright="Q")) + self.write("222.jpg", b"OLD") + self.old = rb.checksum(os.path.join(self.dir, "222.jpg")) + + def conf(self, **over): + conf = {"directory": self.dir, "dry_run": False, "quiet": False, "limit": None} + conf.update(over) + return conf + + def responses(self): + return [ + FakeResponse({"id": "u1"}), # me + FakeResponse({"assets": {"items": [ + {"id": "a2", "ownerId": "u1", "checksum": self.old, + "exifInfo": {"description": "Mine", "latitude": 1, "longitude": 2, + "dateTimeOriginal": "t"}}], "nextPage": None}}), # assets + FakeResponse({"id": "a1", "status": "created"}, status=201), # upload 111 + FakeResponse(b""), # update a2 (append) + FakeResponse([]), # albums + FakeResponse({"id": "al1"}), # create album + FakeResponse({"assets": []}), # album assets + FakeResponse([{"success": True}, {"success": True}]), # add to album + FakeResponse([{"value": "rgl", "id": "t1"}, + {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"}, + {"value": "rgl/photographer/P", "id": "t3"}, + {"value": "rgl/photographer/Q", "id": "t4"}]), # tags + FakeResponse([{"success": True}, {"success": True}]), # t1 + FakeResponse([{"success": True}, {"success": True}]), # t2 + FakeResponse([{"success": True}]), # t3 + FakeResponse([{"success": True}]), # t4 + FakeResponse({"id": "a1", "originalFileName": "111.jpg", + "exifInfo": {}}), # re-read a1 + FakeResponse(b""), # update a1 + ] + + def test_uploads_new_reuses_existing_and_annotates(self): + rb.OPEN = FakeOpen(*self.responses()) + failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 0) + self.assertEqual(rb.STATS["uploaded"], 1) + self.assertEqual(rb.STATS["existing"], 1) + reqs = rb.OPEN.requests + self.assertTrue(reqs[2].full_url.endswith("/assets")) + upd2 = json.loads(reqs[3].data) + self.assertEqual(list(upd2), ["description"]) + self.assertTrue(upd2["description"].startswith("Mine\n\n")) + self.assertEqual(json.loads(reqs[5].data)["albumName"], "RGL") + self.assertEqual(sorted(json.loads(reqs[7].data)["ids"]), ["a1", "a2"]) + # the uploaded asset is annotated last, against what immich now holds + self.assertEqual((reqs[13].get_method(), reqs[13].full_url), + ("GET", "https://i/api/assets/a1")) + upd1 = json.loads(reqs[14].data) + self.assertIn("latitude", upd1) + self.assertEqual(upd1["description"], rb.describeblock(sidecar())) + self.assertTrue(os.path.exists(os.path.join(self.dir, rb.HASHES))) + + def test_duplicate_upload_appends_to_what_immich_already_holds(self): + rb.OPEN = FakeOpen( + FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + FakeResponse({"id": "a1", "status": "duplicate"}), # upload 111 + FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]), + FakeResponse({"assets": []}), + FakeResponse([{"success": True}]), + FakeResponse([{"value": "rgl", "id": "t1"}, + {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"}, + {"value": "rgl/photographer/P", "id": "t3"}]), + FakeResponse([{"success": True}]), + FakeResponse([{"success": True}]), + FakeResponse([{"success": True}]), + FakeResponse({"id": "a1", "originalFileName": "mine.jpg", + "exifInfo": {"description": "My own words", + "latitude": 1, "longitude": 2, + "dateTimeOriginal": "2001-01-01T00:00:00.000Z"}}), + FakeResponse(b"")) + failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 0) + self.assertEqual(rb.STATS["uploaded"], 0) + self.assertEqual(rb.STATS["existing"], 1) + last = rb.OPEN.requests[-1] + self.assertEqual((last.get_method(), last.full_url), + ("PUT", "https://i/api/assets/a1")) + body = json.loads(last.data) + self.assertEqual(list(body), ["description"]) + self.assertTrue(body["description"].startswith("My own words\n\n")) + + def test_a_repeated_checksum_is_uploaded_once(self): + self.write("222.jpg", b"NEW") # byte-identical to 111.jpg + rb.OPEN = FakeOpen( + FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + FakeResponse({"id": "a1", "status": "created"}, status=201), # upload 111 + FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]), + FakeResponse({"assets": [{"id": "a1"}]}), + FakeResponse([{"value": "rgl", "id": "t1"}, + {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"}, + {"value": "rgl/photographer/Q", "id": "t3"}]), + FakeResponse([{"success": True}]), + FakeResponse([{"success": True}]), + FakeResponse([{"success": True}]), + FakeResponse({"id": "a1", "exifInfo": {}}), # re-read a1 + FakeResponse(b"")) + failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 0) + uploads = [r for r in rb.OPEN.requests if r.full_url.endswith("/api/assets")] + self.assertEqual(len(uploads), 1) + self.assertEqual(rb.STATS["uploaded"], 1) + self.assertEqual(rb.STATS["existing"], 1) + + def test_second_run_is_quiet(self): + block1 = rb.describeblock(sidecar()) + block2 = rb.describeblock(sidecar(key="222", file="222.jpg", copyright="Q")) + new = rb.checksum(os.path.join(self.dir, "111.jpg")) + rb.OPEN = FakeOpen( + FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [ + {"id": "a1", "ownerId": "u1", "checksum": new, + "exifInfo": {"description": block1, "latitude": 51.5, "longitude": -0.1, "dateTimeOriginal": "t"}}, + {"id": "a2", "ownerId": "u1", "checksum": self.old, + "exifInfo": {"description": "Mine\n\n" + block2, "latitude": 1, "longitude": 2, "dateTimeOriginal": "t"}}], + "nextPage": None}}), + FakeResponse([{"albumName": "RGL", "id": "al1", "ownerId": "u1"}]), + FakeResponse({"assets": [{"id": "a1"}, {"id": "a2"}]}), + FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"}, + {"value": "rgl/photographer/P", "id": "t3"}, {"value": "rgl/photographer/Q", "id": "t4"}]), + FakeResponse([{"error": "duplicate"}, {"error": "duplicate"}]), + FakeResponse([{"error": "duplicate"}, {"error": "duplicate"}]), + FakeResponse([{"error": "duplicate"}]), + FakeResponse([{"error": "duplicate"}])) + failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 0) + self.assertEqual(rb.STATS["uploaded"], 0) + self.assertEqual(rb.STATS["updated"], 0) + + def test_dry_run_only_reads(self): + rb.OPEN = FakeOpen(FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + FakeResponse([])) + rb.syncphase(self.conf(dry_run=True), rb.Immich("https://i/api", "k")) + self.assertEqual([r.get_method() for r in rb.OPEN.requests], ["GET", "POST", "GET"]) + self.assertIn("would upload", self.stdout.getvalue()) + self.assertFalse(os.path.exists(os.path.join(self.dir, rb.HASHES))) + + def test_dry_run_limit_counts_what_it_looked_at(self): + rb.OPEN = FakeOpen(FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + FakeResponse([])) + rb.syncphase(self.conf(dry_run=True, limit=1), rb.Immich("https://i/api", "k")) + self.assertEqual(self.stdout.getvalue().count("would upload"), 1) + + def test_unsupported_file_is_a_failure_not_a_stop(self): + rb.OPEN = FakeOpen( + FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + http_error(400, b'{"message":"unsupported file type"}'), # upload 111 + FakeResponse({"id": "a2", "status": "created"}, status=201), # upload 222 + FakeResponse([]), FakeResponse({"id": "al1"}), FakeResponse({"assets": []}), + FakeResponse([{"success": True}]), + FakeResponse([{"value": "rgl", "id": "t1"}, {"value": "rgl/licence/cc-by-sa-2.0", "id": "t2"}, + {"value": "rgl/photographer/Q", "id": "t4"}]), + FakeResponse([{"success": True}]), FakeResponse([{"success": True}]), FakeResponse([{"success": True}]), + FakeResponse({"id": "a2", "exifInfo": {}}), # re-read a2 + FakeResponse(b"")) # update a2 + failures = rb.syncphase(self.conf(), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 1) + self.assertIn("unsupported", failures.items[0][2]) + self.assertEqual(rb.STATS["uploaded"], 1) + + def test_malformed_sidecar_is_a_failure_not_a_stop(self): + room = os.path.join(self.dir, "other") + os.mkdir(room) + rb.writejson(os.path.join(room, "333.json"), {"key": "333"}) + rb.OPEN = FakeOpen( + FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + FakeResponse([]), FakeResponse({"id": "al1"}), FakeResponse({"assets": []})) + failures = rb.syncphase(self.conf(directory=room), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 1) + self.assertIn("no file", failures.items[0][2]) + + def test_bad_key_is_fatal_before_writes(self): + rb.OPEN = FakeOpen(http_error(401)) + with self.assertRaises(rb.BackupError): + rb.syncphase(self.conf(), rb.Immich("https://i/api", "k")) + + def test_album_stage_failure_is_recorded(self): + rb.ATTEMPTS = 1 + rb.OPEN = FakeOpen( + FakeResponse({"id": "u1"}), + FakeResponse({"assets": {"items": [], "nextPage": None}}), + FakeResponse({"id": "a1", "status": "created"}, status=201), # upload 111 + http_error(500), # albums + FakeResponse({"id": "a1", "exifInfo": {}}), # re-read a1 + FakeResponse(b"")) # update a1 + failures = rb.syncphase(self.conf(limit=1), rb.Immich("https://i/api", "k")) + self.assertEqual(len(failures), 1) + reason = failures.items[0][2] + self.assertTrue("albums" in reason or "HTTP 500" in reason)