Ask flickr for each photo's details and its largest size

Two unsigned json calls per photo: getInfo for title, owner, dates and
licence, getSizes for the renditions. Sizes come back ordered best-first so
a missing original can fall through to the next. Not-found and permission
errors become PhotoGone so the run can name the wiki pages affected.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

rgl-photo-backup

@@ -488,5 +488,117 @@ def groupphotos(nodes):
     return (groups, rejected)
 
 
+#
+# Flickr answers errors with HTTP 200 and stat="fail", so check the body.
+# RGL's photos are public, so an api key alone is enough: no OAuth.
+#
+# 105 "service currently unavailable" and 106 "write operation failed" are
+# flickr being busy, not this photo being wrong, and they arrive with HTTP 200
+# where retrying() cannot see them, so back off and try again here.
+#
+FLICKR_BUSY = (105, 106)
+
+def flickrcall(method, params, key, context=None):
+    context = context or method
+    query = dict(params)
+    query.update({"method": method, "api_key": key,
+                  "format": "json", "nojsoncallback": "1"})
+    url = REST_URL + "?" + urllib.parse.urlencode(query)
+    for attempt in range(1, ATTEMPTS + 1):
+        body = fetch(url, context, pace=True)
+        try:
+            answer = json.loads(body.decode("utf-8", "replace"))
+        except ValueError:
+            raise BackupError("%s: response was not json: %r" % (context, body[:200]))
+        if answer.get("stat") == "ok":
+            return answer
+        try:
+            code = int(answer.get("code"))
+        except (TypeError, ValueError):
+            code = answer.get("code")
+        message = answer.get("message", "unknown error")
+        # 1 photo not found, 2 permission denied: gone as far as we can tell
+        if code in (1, 2):
+            raise PhotoGone("%s: %s" % (context, message))
+        if code not in FLICKR_BUSY or attempt == ATTEMPTS:
+            raise BackupError("%s: flickr returned error %s: %s" % (context, code, message))
+        wait = backoff(attempt)
+        STATS["retries"] += 1
+        warn("%s: %s - retrying in %.0fs (attempt %d of %d)"
+             % (context, message, wait, attempt + 1, ATTEMPTS))
+        SLEEP(wait)
+
+def isotime(epoch):
+    try:
+        when = datetime.datetime.fromtimestamp(int(epoch), datetime.timezone.utc)
+    except (TypeError, ValueError):
+        return ""
+    return when.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+def getinfo(photoid, key):
+    photo = flickrcall("flickr.photos.getInfo", {"photo_id": photoid}, key,
+                       "photo %s info" % photoid)["photo"]
+    owner = photo.get("owner") or {}
+    page = ""
+    for url in (photo.get("urls") or {}).get("url") or []:
+        if url.get("type") == "photopage":
+            page = url.get("_content", "")
+    try:
+        licence = int(photo.get("license"))
+    except (TypeError, ValueError):
+        licence = None
+    return {
+        "id":              str(photo.get("id") or photoid),
+        "owner":           owner.get("nsid", ""),
+        "owner_name":      owner.get("username", ""),
+        "owner_realname":  owner.get("realname", ""),
+        "title":           (photo.get("title") or {}).get("_content", "").strip(),
+        "description":     (photo.get("description") or {}).get("_content", "").strip(),
+        "taken":           (photo.get("dates") or {}).get("taken", ""),
+        "posted":          isotime((photo.get("dates") or {}).get("posted")),
+        "licence_id":      licence,
+        "page":            page,
+        "original_format": photo.get("originalformat", ""),
+    }
+
+def getsizes(photoid, key):
+    answer = flickrcall("flickr.photos.getSizes", {"photo_id": photoid}, key,
+                        "photo %s sizes" % photoid)
+    sizes = []
+    for size in (answer.get("sizes") or {}).get("size") or []:
+        try:
+            width, height = int(size.get("width")), int(size.get("height"))
+        except (TypeError, ValueError):
+            width = height = 0
+        sizes.append({"label": size.get("label", ""), "width": width, "height": height,
+                      "source": size.get("source", ""), "media": size.get("media") or "photo"})
+    return sizes
+
+#
+# Candidates in the order to try them: the original first, then by area, so a
+# 404 on one size can fall through to the next. Videos are never wiki photos.
+#
+def picksizes(sizes):
+    photos = [s for s in sizes if s["media"] == "photo" and s["source"]
+              and s["label"] != "Video Player"]
+    originals = [s for s in photos if s["label"] == "Original"]
+    rest = sorted((s for s in photos if s["label"] != "Original"),
+                  key=lambda s: s["width"] * s["height"], reverse=True)
+    return originals + rest
+
+IMAGE_EXTENSIONS = (".jpg", ".png", ".gif", ".tif", ".tiff", ".bmp", ".webp")
+CONTENT_EXTENSIONS = {"image/jpeg": ".jpg", "image/png": ".png", "image/gif": ".gif",
+                      "image/tiff": ".tif", "image/webp": ".webp"}
+
+def extensionfor(source, contenttype=""):
+    ext = os.path.splitext(urllib.parse.urlparse(source).path)[1].lower()
+    if ext == ".jpeg":
+        return ".jpg"
+    if ext in IMAGE_EXTENSIONS:
+        return ext
+    kind = (contenttype or "").split(";")[0].strip().lower()
+    return CONTENT_EXTENSIONS.get(kind, ".jpg")
+
+
 if __name__ == "__main__":
     sys.exit(0)

test_rgl_photo_backup.py

@@ -421,6 +421,14 @@ class Retrying(TestCase):
         path = os.path.join(self.dir, "p.part")
         self.assertEqual(rb.download("http://t/p.jpg", "ctx", path), ("image/jpeg", 1000))
         self.assertEqual(os.path.getsize(path), 1000)
+
+    def test_pace_sleeps_before_a_flickr_call(self):
+        waits = []
+        rb.SLEEP = waits.append
+        rb.PACE = 2.0
+        rb.OPEN = FakeOpen(FakeResponse({"stat": "ok"}))
+        rb.flickrcall("flickr.test.echo", {}, "KEY")
+        self.assertEqual(waits, [2.0])
     def test_download_refuses_an_oversized_body(self):
         rb.DOWNLOAD_CAP = 10
         rb.OPEN = FakeOpen(FakeResponse(b"x" * 100))
@@ -429,3 +437,107 @@ class Retrying(TestCase):
             rb.download("http://t/p.jpg", "ctx", path)
         self.assertIn("10", str(cm.exception))
         self.assertEqual(len(rb.OPEN.requests), 1)      # not a retryable failure
+
+    def test_debug_redacts_the_flickr_key(self):
+        rb.DEBUG = True
+        rb.OPEN = FakeOpen(FakeResponse({"stat": "ok"}))
+        rb.flickrcall("flickr.test.echo", {}, "SECRETKEY")
+        logged = self.stderr.getvalue()
+        self.assertIn("api_key=<redacted>", logged)
+        self.assertNotIn("SECRETKEY", logged)
+
+
+def flickr_info(photoid="111", licence=5, realname="Real Name"):
+    return {"stat": "ok", "photo": {
+        "id": photoid, "license": str(licence), "originalformat": "jpg",
+        "owner": {"nsid": "12345678@N00", "username": "handle", "realname": realname},
+        "title": {"_content": "A title"}, "description": {"_content": "Words"},
+        "dates": {"taken": "2009-07-22 14:03:11", "posted": "1248345600"},
+        "urls": {"url": [{"type": "photopage",
+                          "_content": "https://www.flickr.com/photos/12345678@N00/%s/" % photoid}]},
+    }}
+
+def flickr_sizes(*sizes):
+    return {"stat": "ok", "sizes": {"size": [
+        {"label": l, "width": w, "height": h, "source": s, "media": "photo"}
+        for (l, w, h, s) in sizes]}}
+
+
+class Flickr(TestCase):
+    def test_call_builds_unsigned_json_request(self):
+        rb.OPEN = FakeOpen(FakeResponse({"stat": "ok", "x": 1}))
+        self.assertEqual(rb.flickrcall("flickr.test.echo", {"a": "b"}, "KEY")["x"], 1)
+        url = rb.OPEN.requests[0].full_url
+        q = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
+        self.assertEqual(q["method"], ["flickr.test.echo"])
+        self.assertEqual(q["api_key"], ["KEY"])
+        self.assertEqual(q["format"], ["json"])
+        self.assertEqual(q["nojsoncallback"], ["1"])
+        self.assertEqual(q["a"], ["b"])
+
+    def test_api_failure_raises(self):
+        rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": 100, "message": "Invalid API Key"}))
+        with self.assertRaises(rb.BackupError) as cm:
+            rb.flickrcall("flickr.photos.getInfo", {"photo_id": "1"}, "k")
+        self.assertIn("Invalid API Key", str(cm.exception))
+
+    def test_transient_service_error_is_retried(self):
+        busy = {"stat": "fail", "code": 105, "message": "Service currently unavailable"}
+        rb.OPEN = FakeOpen(FakeResponse(busy), FakeResponse(busy),
+                           FakeResponse({"stat": "ok", "x": 1}))
+        self.assertEqual(rb.flickrcall("flickr.test.echo", {}, "k")["x"], 1)
+        self.assertEqual(len(rb.OPEN.requests), 3)
+        self.assertEqual(rb.STATS["retries"], 2)
+
+    def test_transient_service_error_gives_up_after_attempts(self):
+        rb.ATTEMPTS = 2
+        busy = {"stat": "fail", "code": "106", "message": "Write operation failed"}
+        rb.OPEN = FakeOpen(FakeResponse(busy), FakeResponse(busy))
+        with self.assertRaises(rb.BackupError) as cm:
+            rb.flickrcall("flickr.test.echo", {}, "k")
+        self.assertIn("Write operation failed", str(cm.exception))
+        self.assertEqual(len(rb.OPEN.requests), 2)
+
+    def test_missing_or_private_photo_is_gone(self):
+        for code in (1, 2):
+            rb.OPEN = FakeOpen(FakeResponse({"stat": "fail", "code": code, "message": "nope"}))
+            with self.assertRaises(rb.PhotoGone):
+                rb.getinfo("1", "k")
+
+    def test_getinfo_flattens(self):
+        rb.OPEN = FakeOpen(FakeResponse(flickr_info()))
+        info = rb.getinfo("111", "k")
+        self.assertEqual(info["id"], "111")
+        self.assertEqual(info["owner"], "12345678@N00")
+        self.assertEqual(info["owner_name"], "handle")
+        self.assertEqual(info["owner_realname"], "Real Name")
+        self.assertEqual(info["title"], "A title")
+        self.assertEqual(info["taken"], "2009-07-22 14:03:11")
+        self.assertEqual(info["posted"], "2009-07-23T10:40:00Z")
+        self.assertEqual(info["licence_id"], 5)
+        self.assertEqual(info["page"], "https://www.flickr.com/photos/12345678@N00/111/")
+        self.assertEqual(info["original_format"], "jpg")
+
+    def test_getsizes_normalises_numbers(self):
+        rb.OPEN = FakeOpen(FakeResponse(flickr_sizes(("Small", "240", "180", "https://x/s.jpg"))))
+        sizes = rb.getsizes("111", "k")
+        self.assertEqual(sizes[0]["width"], 240)
+        self.assertEqual(sizes[0]["label"], "Small")
+
+    def test_picksizes_prefers_original_then_area(self):
+        sizes = [
+            {"label": "Small", "width": 240, "height": 180, "source": "s", "media": "photo"},
+            {"label": "Large", "width": 1024, "height": 768, "source": "l", "media": "photo"},
+            {"label": "Original", "width": 800, "height": 600, "source": "o", "media": "photo"},
+            {"label": "Video Player", "width": 640, "height": 480, "source": "v", "media": "video"},
+        ]
+        self.assertEqual([s["label"] for s in rb.picksizes(sizes)], ["Original", "Large", "Small"])
+
+    def test_picksizes_empty(self):
+        self.assertEqual(rb.picksizes([]), [])
+
+    def test_extensionfor(self):
+        self.assertEqual(rb.extensionfor("https://x/1_o.png", ""), ".png")
+        self.assertEqual(rb.extensionfor("https://x/1_o.jpeg", ""), ".jpg")
+        self.assertEqual(rb.extensionfor("https://x/1_o", "image/gif"), ".gif")
+        self.assertEqual(rb.extensionfor("https://x/1_o.weird", "text/html"), ".jpg")