Read the RGL index and group nodes by photo

One json request to the wiki's index carries every node's image fields, so
no per-page scraping is needed. Nodes are grouped by flickr photo id, taken
from the thumbnail filename or the photo page, so a photo used on several
pages is fetched once and its description can list them all.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

rgl-photo-backup

@@ -378,5 +378,115 @@ def download(url, context, path, pace=False):
     return retrying(url, context, consume, pace=pace)
 
 
+#
+# The wiki's index in json: one entry per node, metadata values wrapped in
+# one-element lists. It is not valid utf-8 throughout, so decode leniently
+# rather than lose the whole run to one bad byte.
+#
+def nodeurl(param):
+    return NODE_URL % urllib.parse.quote(param, safe="_.-~()!*'")
+
+def readindex(raw):
+    text = raw.decode("utf-8", "replace")
+    try:
+        entries = json.loads(text)
+    except ValueError as e:
+        raise BackupError("index was not valid json: %s" % e)
+    nodes = []
+    for entry in entries:
+        md = (entry.get("node_data") or {}).get("metadata") or {}
+        def first(name):
+            values = md.get(name) or [""]
+            return (values[0] or "").strip()
+        def number(name):                          # 0 is a coordinate, not a blank
+            value = entry.get(name)
+            return "" if value is None else str(value)
+        param = entry.get("param") or ""
+        nodes.append({
+            "param":     param,
+            "name":      entry.get("name") or param.replace("_", " "),
+            "url":       nodeurl(param),
+            "image":     first("node_image"),
+            "page":      first("node_image_url"),
+            "copyright": first("node_image_copyright"),
+            "licence":   first("node_image_licence"),
+            "lat":       number("wgs84_lat"),
+            "long":      number("wgs84_long"),
+        })
+    return nodes
+
+def loadindex(url):
+    return readindex(fetch(url, "index"))
+
+
+BASE58 = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"
+
+def base58decode(text):
+    value = 0
+    for ch in text:
+        value = value * 58 + BASE58.index(ch)
+    return value
+
+FLICKR_HOSTS = re.compile(r"(^|\.)(flickr\.com|staticflickr\.com|static\.flickr\.com)$")
+FLICKR_FILE  = re.compile(r"^(\d+)_[0-9a-f]+(_[a-z0-9]+)?\.\w+$")
+FLICKR_PAGE  = re.compile(r"/photos/[^/]+/(\d+)")
+FLICKR_SHORT = re.compile(r"^/p/([1-9A-HJ-NP-Za-km-z]+)")
+
+#
+# What identifies a photo: the flickr id where there is one, otherwise a hash
+# of the image url. The thumbnail filename carries the id and is the most
+# reliable source; the photo page is the fallback, including flic.kr links.
+#
+def photokey(node):
+    image = node.get("image") or ""
+    parts = urllib.parse.urlparse(image)
+    if parts.scheme not in ("http", "https") or not parts.netloc:
+        return None
+
+    if FLICKR_HOSTS.search(parts.netloc):
+        found = FLICKR_FILE.match(os.path.basename(parts.path))
+        if found:
+            return ("flickr", found.group(1))
+
+    page = urllib.parse.urlparse(node.get("page") or "")
+    if page.netloc.endswith("flickr.com"):
+        found = FLICKR_PAGE.search(page.path)
+        if found:
+            return ("flickr", found.group(1))
+    if page.netloc == "flic.kr":
+        found = FLICKR_SHORT.match(page.path)
+        if found:
+            return ("flickr", str(base58decode(found.group(1))))
+
+    digest = hashlib.sha1(image.encode("utf-8")).hexdigest()[:16]
+    return ("url", "url-" + digest)
+
+def groupphotos(nodes):
+    groups = {}
+    rejected = []
+    for node in nodes:
+        if not node["image"]:
+            continue
+        key = photokey(node)
+        if key is None:
+            rejected.append((node, "node_image is not a url: %r" % node["image"]))
+            continue
+        source, ident = key
+        group = groups.get(ident)
+        if group is None:
+            group = groups[ident] = {
+                "key": ident, "source": source,
+                "image": node["image"], "page": node["page"],
+                "copyright": "", "licence": "", "nodes": [],
+            }
+            if source == "flickr":
+                group["id"] = ident
+        group["nodes"].append(node)
+        for field in ("copyright", "licence", "page"):
+            if node[field] and not group[field]:
+                group[field] = node[field]
+    return (groups, rejected)
+
+
 if __name__ == "__main__":
     sys.exit(0)

test_rgl_photo_backup.py

@@ -105,6 +105,132 @@ class TestCase(unittest.TestCase):
         return path
 
 
+def index_entry(param, image, page="", copyright="", licence="", lat="51.5", long="-0.1", name=None):
+    md = {}
+    if image:
+        md["node_image"] = [image]
+    if page:
+        md["node_image_url"] = [page]
+    if copyright:
+        md["node_image_copyright"] = [copyright]
+    if licence:
+        md["node_image_licence"] = [licence]
+    return {"param": param, "name": name or param.replace("_", " "),
+            "wgs84_lat": lat, "wgs84_long": long, "has_geodata": 1,
+            "node_data": {"metadata": md}}
+
+
+class Index(TestCase):
+    def test_readindex_tolerates_invalid_utf8(self):
+        entry = index_entry("Café,_N1", "https://farm1.static.flickr.com/1/123_abc_m.jpg")
+        raw = json.dumps([entry], ensure_ascii=False).encode("utf-8").replace("\xe9".encode(), b"\xc2")
+        nodes = rb.readindex(raw)
+        self.assertEqual(len(nodes), 1)
+        self.assertIn("�", nodes[0]["param"])
+
+    def test_readindex_flattens_fields(self):
+        raw = json.dumps([index_entry("Pub,_E1", "https://farm1.static.flickr.com/1/123_abc_m.jpg",
+                                      page="https://www.flickr.com/photos/x/123/",
+                                      copyright="Someone", licence="https://creativecommons.org/licenses/by-sa/2.0/"),
+                          index_entry("Nopic,_E2", "")]).encode()
+        nodes = rb.readindex(raw)
+        self.assertEqual(len(nodes), 2)
+        n = nodes[0]
+        self.assertEqual(n["param"], "Pub,_E1")
+        self.assertEqual(n["name"], "Pub, E1")
+        self.assertEqual(n["image"], "https://farm1.static.flickr.com/1/123_abc_m.jpg")
+        self.assertEqual(n["page"], "https://www.flickr.com/photos/x/123/")
+        self.assertEqual(n["copyright"], "Someone")
+        self.assertEqual(n["licence"], "https://creativecommons.org/licenses/by-sa/2.0/")
+        self.assertEqual((n["lat"], n["long"]), ("51.5", "-0.1"))
+        self.assertEqual(n["url"], "https://london.randomness.org.uk/wiki.cgi?Pub%2C_E1")
+        self.assertEqual(nodes[1]["image"], "")
+
+    def test_zero_coordinates_are_kept(self):
+        raw = json.dumps([index_entry("Null,_Island", "https://x/1_a_m.jpg", lat=0, long=0)]).encode()
+        node = rb.readindex(raw)[0]
+        self.assertEqual((node["lat"], node["long"]), ("0", "0"))
+
+    def test_missing_coordinates_are_empty(self):
+        raw = json.dumps([index_entry("Nowhere", "https://x/1_a_m.jpg", lat=None, long=None)]).encode()
+        node = rb.readindex(raw)[0]
+        self.assertEqual((node["lat"], node["long"]), ("", ""))
+
+    def test_loadindex_fetches(self):
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps([index_entry("A", "")]).encode()))
+        self.assertEqual(len(rb.loadindex("http://t/index")), 1)
+
+
+class PhotoKeys(TestCase):
+    def key(self, image, page=""):
+        return rb.photokey({"image": image, "page": page, "param": "X"})
+
+    def test_id_from_thumbnail_filename(self):
+        self.assertEqual(self.key("https://farm4.static.flickr.com/3188/3002966579_97596aea2e_m.jpg"),
+                         ("flickr", "3002966579"))
+        self.assertEqual(self.key("https://live.staticflickr.com/65535/52123_abc_m.jpg"),
+                         ("flickr", "52123"))
+
+    def test_id_from_page_when_thumbnail_is_odd(self):
+        self.assertEqual(self.key("https://www.flickr.com/some/other.jpg",
+                                  "https://www.flickr.com/photos/kake/3002966579/in/set-1/"),
+                         ("flickr", "3002966579"))
+        self.assertEqual(self.key("https://c1.staticflickr.com/x.jpg",
+                                  "https://secure.flickr.com/photos/12345678@N00/98765"),
+                         ("flickr", "98765"))
+
+    def test_short_link_is_base58(self):
+        self.assertEqual(rb.base58decode("5zmZkR"), 3002966579)
+        self.assertEqual(self.key("https://c1.staticflickr.com/x.jpg", "https://flic.kr/p/5zmZkR"),
+                         ("flickr", "3002966579"))
+
+    def test_non_flickr_hosts_key_on_url_hash(self):
+        source, key = self.key("https://www.geograph.org.uk/photos/12/34/1234.jpg",
+                               "https://www.geograph.org.uk/photo/1234")
+        self.assertEqual(source, "url")
+        self.assertRegex(key, r"^url-[0-9a-f]{16}$")
+
+    def test_flickr_cdn_without_id_falls_back_to_url_key(self):
+        source, key = self.key("https://c1.staticflickr.com/nodigits.jpg")
+        self.assertEqual(source, "url")
+
+    def test_not_a_url_is_none(self):
+        self.assertIsNone(self.key("not a url"))
+        self.assertIsNone(self.key(""))
+
+
+class Grouping(TestCase):
+    def test_nodes_sharing_a_photo_group_together(self):
+        nodes = rb.readindex(json.dumps([
+            index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                        page="https://www.flickr.com/photos/x/111/", copyright="P", licence="L"),
+            index_entry("B,_E2", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                        page="https://www.flickr.com/photos/x/111/"),
+            index_entry("C,_E3", "https://farm1.static.flickr.com/1/222_b_m.jpg"),
+            index_entry("D,_E4", ""),
+            index_entry("E,_E5", "junk"),
+        ]).encode())
+        groups, rejected = rb.groupphotos(nodes)
+        self.assertEqual(sorted(groups), ["111", "222"])
+        g = groups["111"]
+        self.assertEqual(g["source"], "flickr")
+        self.assertEqual(g["id"], "111")
+        self.assertEqual([n["param"] for n in g["nodes"]], ["A,_E1", "B,_E2"])
+        self.assertEqual(g["copyright"], "P")
+        self.assertEqual(g["licence"], "L")
+        self.assertEqual(g["page"], "https://www.flickr.com/photos/x/111/")
+        self.assertEqual(len(rejected), 1)
+        self.assertEqual(rejected[0][0]["param"], "E,_E5")
+
+    def test_first_non_empty_copyright_wins(self):
+        nodes = rb.readindex(json.dumps([
+            index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"),
+            index_entry("B", "https://farm1.static.flickr.com/1/111_a_m.jpg", copyright="Later"),
+        ]).encode())
+        groups, _ = rb.groupphotos(nodes)
+        self.assertEqual(groups["111"]["copyright"], "Later")
+
+
 class Output(TestCase):
     def test_say_prints_unless_quiet(self):
         rb.QUIET = False