Read the RGL index and group nodes by photo

One json request to the wiki's index carries every node's image fields, so
no per-page scraping is needed. Nodes are grouped by flickr photo id, taken
from the thumbnail filename or the photo page, so a photo used on several
pages is fetched once and its description can list them all.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

test_rgl_photo_backup.py

@@ -105,6 +105,132 @@ class TestCase(unittest.TestCase):
         return path
 
 
+def index_entry(param, image, page="", copyright="", licence="", lat="51.5", long="-0.1", name=None):
+    md = {}
+    if image:
+        md["node_image"] = [image]
+    if page:
+        md["node_image_url"] = [page]
+    if copyright:
+        md["node_image_copyright"] = [copyright]
+    if licence:
+        md["node_image_licence"] = [licence]
+    return {"param": param, "name": name or param.replace("_", " "),
+            "wgs84_lat": lat, "wgs84_long": long, "has_geodata": 1,
+            "node_data": {"metadata": md}}
+
+
+class Index(TestCase):
+    def test_readindex_tolerates_invalid_utf8(self):
+        entry = index_entry("Café,_N1", "https://farm1.static.flickr.com/1/123_abc_m.jpg")
+        raw = json.dumps([entry], ensure_ascii=False).encode("utf-8").replace("\xe9".encode(), b"\xc2")
+        nodes = rb.readindex(raw)
+        self.assertEqual(len(nodes), 1)
+        self.assertIn("�", nodes[0]["param"])
+
+    def test_readindex_flattens_fields(self):
+        raw = json.dumps([index_entry("Pub,_E1", "https://farm1.static.flickr.com/1/123_abc_m.jpg",
+                                      page="https://www.flickr.com/photos/x/123/",
+                                      copyright="Someone", licence="https://creativecommons.org/licenses/by-sa/2.0/"),
+                          index_entry("Nopic,_E2", "")]).encode()
+        nodes = rb.readindex(raw)
+        self.assertEqual(len(nodes), 2)
+        n = nodes[0]
+        self.assertEqual(n["param"], "Pub,_E1")
+        self.assertEqual(n["name"], "Pub, E1")
+        self.assertEqual(n["image"], "https://farm1.static.flickr.com/1/123_abc_m.jpg")
+        self.assertEqual(n["page"], "https://www.flickr.com/photos/x/123/")
+        self.assertEqual(n["copyright"], "Someone")
+        self.assertEqual(n["licence"], "https://creativecommons.org/licenses/by-sa/2.0/")
+        self.assertEqual((n["lat"], n["long"]), ("51.5", "-0.1"))
+        self.assertEqual(n["url"], "https://london.randomness.org.uk/wiki.cgi?Pub%2C_E1")
+        self.assertEqual(nodes[1]["image"], "")
+
+    def test_zero_coordinates_are_kept(self):
+        raw = json.dumps([index_entry("Null,_Island", "https://x/1_a_m.jpg", lat=0, long=0)]).encode()
+        node = rb.readindex(raw)[0]
+        self.assertEqual((node["lat"], node["long"]), ("0", "0"))
+
+    def test_missing_coordinates_are_empty(self):
+        raw = json.dumps([index_entry("Nowhere", "https://x/1_a_m.jpg", lat=None, long=None)]).encode()
+        node = rb.readindex(raw)[0]
+        self.assertEqual((node["lat"], node["long"]), ("", ""))
+
+    def test_loadindex_fetches(self):
+        rb.OPEN = FakeOpen(FakeResponse(json.dumps([index_entry("A", "")]).encode()))
+        self.assertEqual(len(rb.loadindex("http://t/index")), 1)
+
+
+class PhotoKeys(TestCase):
+    def key(self, image, page=""):
+        return rb.photokey({"image": image, "page": page, "param": "X"})
+
+    def test_id_from_thumbnail_filename(self):
+        self.assertEqual(self.key("https://farm4.static.flickr.com/3188/3002966579_97596aea2e_m.jpg"),
+                         ("flickr", "3002966579"))
+        self.assertEqual(self.key("https://live.staticflickr.com/65535/52123_abc_m.jpg"),
+                         ("flickr", "52123"))
+
+    def test_id_from_page_when_thumbnail_is_odd(self):
+        self.assertEqual(self.key("https://www.flickr.com/some/other.jpg",
+                                  "https://www.flickr.com/photos/kake/3002966579/in/set-1/"),
+                         ("flickr", "3002966579"))
+        self.assertEqual(self.key("https://c1.staticflickr.com/x.jpg",
+                                  "https://secure.flickr.com/photos/12345678@N00/98765"),
+                         ("flickr", "98765"))
+
+    def test_short_link_is_base58(self):
+        self.assertEqual(rb.base58decode("5zmZkR"), 3002966579)
+        self.assertEqual(self.key("https://c1.staticflickr.com/x.jpg", "https://flic.kr/p/5zmZkR"),
+                         ("flickr", "3002966579"))
+
+    def test_non_flickr_hosts_key_on_url_hash(self):
+        source, key = self.key("https://www.geograph.org.uk/photos/12/34/1234.jpg",
+                               "https://www.geograph.org.uk/photo/1234")
+        self.assertEqual(source, "url")
+        self.assertRegex(key, r"^url-[0-9a-f]{16}$")
+
+    def test_flickr_cdn_without_id_falls_back_to_url_key(self):
+        source, key = self.key("https://c1.staticflickr.com/nodigits.jpg")
+        self.assertEqual(source, "url")
+
+    def test_not_a_url_is_none(self):
+        self.assertIsNone(self.key("not a url"))
+        self.assertIsNone(self.key(""))
+
+
+class Grouping(TestCase):
+    def test_nodes_sharing_a_photo_group_together(self):
+        nodes = rb.readindex(json.dumps([
+            index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                        page="https://www.flickr.com/photos/x/111/", copyright="P", licence="L"),
+            index_entry("B,_E2", "https://farm1.static.flickr.com/1/111_a_m.jpg",
+                        page="https://www.flickr.com/photos/x/111/"),
+            index_entry("C,_E3", "https://farm1.static.flickr.com/1/222_b_m.jpg"),
+            index_entry("D,_E4", ""),
+            index_entry("E,_E5", "junk"),
+        ]).encode())
+        groups, rejected = rb.groupphotos(nodes)
+        self.assertEqual(sorted(groups), ["111", "222"])
+        g = groups["111"]
+        self.assertEqual(g["source"], "flickr")
+        self.assertEqual(g["id"], "111")
+        self.assertEqual([n["param"] for n in g["nodes"]], ["A,_E1", "B,_E2"])
+        self.assertEqual(g["copyright"], "P")
+        self.assertEqual(g["licence"], "L")
+        self.assertEqual(g["page"], "https://www.flickr.com/photos/x/111/")
+        self.assertEqual(len(rejected), 1)
+        self.assertEqual(rejected[0][0]["param"], "E,_E5")
+
+    def test_first_non_empty_copyright_wins(self):
+        nodes = rb.readindex(json.dumps([
+            index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"),
+            index_entry("B", "https://farm1.static.flickr.com/1/111_a_m.jpg", copyright="Later"),
+        ]).encode())
+        groups, _ = rb.groupphotos(nodes)
+        self.assertEqual(groups["111"]["copyright"], "Later")
+
+
 class Output(TestCase):
     def test_say_prints_unless_quiet(self):
         rb.QUIET = False