From fc434a3d9bf9e18793c72b67636bfc719639e114 Mon Sep 17 00:00:00 2001 From: bob walker Date: Thu, 10 Sep 2026 12:07:44 +0100 Subject: [PATCH] Ask flickr for each photo's details and its largest size Two unsigned json calls per photo: getInfo for title, owner, dates and licence, getSizes for the renditions. Sizes come back ordered best-first so a missing original can fall through to the next. Not-found and permission errors become PhotoGone so the run can name the wiki pages affected. Co-Authored-By: Claude Fable 5.1 --- rgl-photo-backup | 112 +++++++++++++++++++++++++++++++++++++++ test_rgl_photo_backup.py | 112 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/rgl-photo-backup b/rgl-photo-backup index fe62f69..408a4aa 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -488,5 +488,117 @@ def groupphotos(nodes): return (groups, rejected) +# +# Flickr answers errors with HTTP 200 and stat="fail", so check the body. +# RGL's photos are public, so an api key alone is enough: no OAuth. +# +# 105 "service currently unavailable" and 106 "write operation failed" are +# flickr being busy, not this photo being wrong, and they arrive with HTTP 200 +# where retrying() cannot see them, so back off and try again here. +# +FLICKR_BUSY = (105, 106) + +def flickrcall(method, params, key, context=None): + context = context or method + query = dict(params) + query.update({"method": method, "api_key": key, + "format": "json", "nojsoncallback": "1"}) + url = REST_URL + "?" + urllib.parse.urlencode(query) + for attempt in range(1, ATTEMPTS + 1): + body = fetch(url, context, pace=True) + try: + answer = json.loads(body.decode("utf-8", "replace")) + except ValueError: + raise BackupError("%s: response was not json: %r" % (context, body[:200])) + if answer.get("stat") == "ok": + return answer + try: + code = int(answer.get("code")) + except (TypeError, ValueError): + code = answer.get("code") + message = answer.get("message", "unknown error") + # 1 photo not found, 2 permission denied: gone as far as we can tell + if code in (1, 2): + raise PhotoGone("%s: %s" % (context, message)) + if code not in FLICKR_BUSY or attempt == ATTEMPTS: + raise BackupError("%s: flickr returned error %s: %s" % (context, code, message)) + wait = backoff(attempt) + STATS["retries"] += 1 + warn("%s: %s - retrying in %.0fs (attempt %d of %d)" + % (context, message, wait, attempt + 1, ATTEMPTS)) + SLEEP(wait) + +def isotime(epoch): + try: + when = datetime.datetime.fromtimestamp(int(epoch), datetime.timezone.utc) + except (TypeError, ValueError): + return "" + return when.strftime("%Y-%m-%dT%H:%M:%SZ") + +def getinfo(photoid, key): + photo = flickrcall("flickr.photos.getInfo", {"photo_id": photoid}, key, + "photo %s info" % photoid)["photo"] + owner = photo.get("owner") or {} + page = "" + for url in (photo.get("urls") or {}).get("url") or []: + if url.get("type") == "photopage": + page = url.get("_content", "") + try: + licence = int(photo.get("license")) + except (TypeError, ValueError): + licence = None + return { + "id": str(photo.get("id") or photoid), + "owner": owner.get("nsid", ""), + "owner_name": owner.get("username", ""), + "owner_realname": owner.get("realname", ""), + "title": (photo.get("title") or {}).get("_content", "").strip(), + "description": (photo.get("description") or {}).get("_content", "").strip(), + "taken": (photo.get("dates") or {}).get("taken", ""), + "posted": isotime((photo.get("dates") or {}).get("posted")), + "licence_id": licence, + "page": page, + "original_format": photo.get("originalformat", ""), + } + +def getsizes(photoid, key): + answer = flickrcall("flickr.photos.getSizes", {"photo_id": photoid}, key, + "photo %s sizes" % photoid) + sizes = [] + for size in (answer.get("sizes") or {}).get("size") or []: + try: + width, height = int(size.get("width")), int(size.get("height")) + except (TypeError, ValueError): + width = height = 0 + sizes.append({"label": size.get("label", ""), "width": width, "height": height, + "source": size.get("source", ""), "media": size.get("media") or "photo"}) + return sizes + +# +# Candidates in the order to try them: the original first, then by area, so a +# 404 on one size can fall through to the next. Videos are never wiki photos. +# +def picksizes(sizes): + photos = [s for s in sizes if s["media"] == "photo" and s["source"] + and s["label"] != "Video Player"] + originals = [s for s in photos if s["label"] == "Original"] + rest = sorted((s for s in photos if s["label"] != "Original"), + key=lambda s: s["width"] * s["height"], reverse=True) + return originals + rest + +IMAGE_EXTENSIONS = (".jpg", ".png", ".gif", ".tif", ".tiff", ".bmp", ".webp") +CONTENT_EXTENSIONS = {"image/jpeg": ".jpg", "image/png": ".png", "image/gif": ".gif", + "image/tiff": ".tif", "image/webp": ".webp"} + +def extensionfor(source, contenttype=""): + ext = os.path.splitext(urllib.parse.urlparse(source).path)[1].lower() + if ext == ".jpeg": + return ".jpg" + if ext in IMAGE_EXTENSIONS: + return ext + kind = (contenttype or "").split(";")[0].strip().lower() + return CONTENT_EXTENSIONS.get(kind, ".jpg") + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index f49b068..ffbb06a 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -421,6 +421,14 @@ class Retrying(TestCase): path = os.path.join(self.dir, "p.part") self.assertEqual(rb.download("http://t/p.jpg", "ctx", path), ("image/jpeg", 1000)) self.assertEqual(os.path.getsize(path), 1000) + + def test_pace_sleeps_before_a_flickr_call(self): + waits = [] + rb.SLEEP = waits.append + rb.PACE = 2.0 + rb.OPEN = FakeOpen(FakeResponse({"stat": "ok"})) + rb.flickrcall("flickr.test.echo", {}, "KEY") + self.assertEqual(waits, [2.0]) def test_download_refuses_an_oversized_body(self): rb.DOWNLOAD_CAP = 10 rb.OPEN = FakeOpen(FakeResponse(b"x" * 100)) @@ -429,3 +437,107 @@ class Retrying(TestCase): rb.download("http://t/p.jpg", "ctx", path) self.assertIn("10", str(cm.exception)) self.assertEqual(len(rb.OPEN.requests), 1) # not a retryable failure + + def test_debug_redacts_the_flickr_key(self): + rb.DEBUG = True + rb.OPEN = FakeOpen(FakeResponse({"stat": "ok"})) + rb.flickrcall("flickr.test.echo", {}, "SECRETKEY") + logged = self.stderr.getvalue() + self.assertIn("api_key=", logged) + self.assertNotIn("SECRETKEY", logged) + + +def flickr_info(photoid="111", licence=5, realname="Real Name"): + return {"stat": "ok", "photo": { + "id": photoid, "license": str(licence), "originalformat": "jpg", + "owner": {"nsid": "12345678@N00", "username": "handle", "realname": realname}, + "title": {"_content": "A title"}, "description": {"_content": "Words"}, + "dates": {"taken": "2009-07-22 14:03:11", "posted": "1248345600"}, + "urls": {"url": [{"type": "photopage", + "_content": "https://www.flickr.com/photos/12345678@N00/%s/" % photoid}]}, + }} + +def flickr_sizes(*sizes): + return {"stat": "ok", "sizes": {"size": [ + {"label": l, "width": w, "height": h, "source": s, "media": "photo"} + for (l, w, h, s) in sizes]}} + + +class Flickr(TestCase): + def test_call_builds_unsigned_json_request(self): + rb.OPEN = FakeOpen(FakeResponse({"stat": "ok", "x": 1})) + self.assertEqual(rb.flickrcall("flickr.test.echo", {"a": "b"}, "KEY")["x"], 1) + url = rb.OPEN.requests[0].full_url + q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query) + self.assertEqual(q["method"], ["flickr.test.echo"]) + self.assertEqual(q["api_key"], ["KEY"]) + self.assertEqual(q["format"], ["json"]) + self.assertEqual(q["nojsoncallback"], ["1"]) + self.assertEqual(q["a"], ["b"]) + + def test_api_failure_raises(self): + rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": 100, "message": "Invalid API Key"})) + with self.assertRaises(rb.BackupError) as cm: + rb.flickrcall("flickr.photos.getInfo", {"photo_id": "1"}, "k") + self.assertIn("Invalid API Key", str(cm.exception)) + + def test_transient_service_error_is_retried(self): + busy = {"stat": "fail", "code": 105, "message": "Service currently unavailable"} + rb.OPEN = FakeOpen(FakeResponse(busy), FakeResponse(busy), + FakeResponse({"stat": "ok", "x": 1})) + self.assertEqual(rb.flickrcall("flickr.test.echo", {}, "k")["x"], 1) + self.assertEqual(len(rb.OPEN.requests), 3) + self.assertEqual(rb.STATS["retries"], 2) + + def test_transient_service_error_gives_up_after_attempts(self): + rb.ATTEMPTS = 2 + busy = {"stat": "fail", "code": "106", "message": "Write operation failed"} + rb.OPEN = FakeOpen(FakeResponse(busy), FakeResponse(busy)) + with self.assertRaises(rb.BackupError) as cm: + rb.flickrcall("flickr.test.echo", {}, "k") + self.assertIn("Write operation failed", str(cm.exception)) + self.assertEqual(len(rb.OPEN.requests), 2) + + def test_missing_or_private_photo_is_gone(self): + for code in (1, 2): + rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": code, "message": "nope"})) + with self.assertRaises(rb.PhotoGone): + rb.getinfo("1", "k") + + def test_getinfo_flattens(self): + rb.OPEN = FakeOpen(FakeResponse(flickr_info())) + info = rb.getinfo("111", "k") + self.assertEqual(info["id"], "111") + self.assertEqual(info["owner"], "12345678@N00") + self.assertEqual(info["owner_name"], "handle") + self.assertEqual(info["owner_realname"], "Real Name") + self.assertEqual(info["title"], "A title") + self.assertEqual(info["taken"], "2009-07-22 14:03:11") + self.assertEqual(info["posted"], "2009-07-23T10:40:00Z") + self.assertEqual(info["licence_id"], 5) + self.assertEqual(info["page"], "https://www.flickr.com/photos/12345678@N00/111/") + self.assertEqual(info["original_format"], "jpg") + + def test_getsizes_normalises_numbers(self): + rb.OPEN = FakeOpen(FakeResponse(flickr_sizes(("Small", "240", "180", "https://x/s.jpg")))) + sizes = rb.getsizes("111", "k") + self.assertEqual(sizes[0]["width"], 240) + self.assertEqual(sizes[0]["label"], "Small") + + def test_picksizes_prefers_original_then_area(self): + sizes = [ + {"label": "Small", "width": 240, "height": 180, "source": "s", "media": "photo"}, + {"label": "Large", "width": 1024, "height": 768, "source": "l", "media": "photo"}, + {"label": "Original", "width": 800, "height": 600, "source": "o", "media": "photo"}, + {"label": "Video Player", "width": 640, "height": 480, "source": "v", "media": "video"}, + ] + self.assertEqual([s["label"] for s in rb.picksizes(sizes)], ["Original", "Large", "Small"]) + + def test_picksizes_empty(self): + self.assertEqual(rb.picksizes([]), []) + + def test_extensionfor(self): + self.assertEqual(rb.extensionfor("https://x/1_o.png", ""), ".png") + self.assertEqual(rb.extensionfor("https://x/1_o.jpeg", ""), ".jpg") + self.assertEqual(rb.extensionfor("https://x/1_o", "image/gif"), ".gif") + self.assertEqual(rb.extensionfor("https://x/1_o.weird", "text/html"), ".jpg")