From b68d4779380d5e4bab2fe90f540f867f8c1c6366 Mon Sep 17 00:00:00 2001 From: bob walker Date: Thu, 10 Sep 2026 12:07:44 +0100 Subject: [PATCH] Read the RGL index and group nodes by photo One json request to the wiki's index carries every node's image fields, so no per-page scraping is needed. Nodes are grouped by flickr photo id, taken from the thumbnail filename or the photo page, so a photo used on several pages is fetched once and its description can list them all. Co-Authored-By: Claude Fable 5.1 --- rgl-photo-backup | 110 ++++++++++++++++++++++++++++++++++ test_rgl_photo_backup.py | 126 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) diff --git a/rgl-photo-backup b/rgl-photo-backup index 0bd9fa4..fe62f69 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -378,5 +378,115 @@ def download(url, context, path, pace=False): return retrying(url, context, consume, pace=pace) +# +# The wiki's index in json: one entry per node, metadata values wrapped in +# one-element lists. It is not valid utf-8 throughout, so decode leniently +# rather than lose the whole run to one bad byte. +# +def nodeurl(param): + return NODE_URL % urllib.parse.quote(param, safe="_.-~()!*'") + +def readindex(raw): + text = raw.decode("utf-8", "replace") + try: + entries = json.loads(text) + except ValueError as e: + raise BackupError("index was not valid json: %s" % e) + nodes = [] + for entry in entries: + md = (entry.get("node_data") or {}).get("metadata") or {} + def first(name): + values = md.get(name) or [""] + return (values[0] or "").strip() + def number(name): # 0 is a coordinate, not a blank + value = entry.get(name) + return "" if value is None else str(value) + param = entry.get("param") or "" + nodes.append({ + "param": param, + "name": entry.get("name") or param.replace("_", " "), + "url": nodeurl(param), + "image": first("node_image"), + "page": first("node_image_url"), + "copyright": first("node_image_copyright"), + "licence": first("node_image_licence"), + "lat": number("wgs84_lat"), + "long": number("wgs84_long"), + }) + return nodes + +def loadindex(url): + return readindex(fetch(url, "index")) + + +BASE58 = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ" + +def base58decode(text): + value = 0 + for ch in text: + value = value * 58 + BASE58.index(ch) + return value + +FLICKR_HOSTS = re.compile(r"(^|\.)(flickr\.com|staticflickr\.com|static\.flickr\.com)$") +FLICKR_FILE = re.compile(r"^(\d+)_[0-9a-f]+(_[a-z0-9]+)?\.\w+$") +FLICKR_PAGE = re.compile(r"/photos/[^/]+/(\d+)") +FLICKR_SHORT = re.compile(r"^/p/([1-9A-HJ-NP-Za-km-z]+)") + +# +# What identifies a photo: the flickr id where there is one, otherwise a hash +# of the image url. The thumbnail filename carries the id and is the most +# reliable source; the photo page is the fallback, including flic.kr links. +# +def photokey(node): + image = node.get("image") or "" + parts = urllib.parse.urlparse(image) + if parts.scheme not in ("http", "https") or not parts.netloc: + return None + + if FLICKR_HOSTS.search(parts.netloc): + found = FLICKR_FILE.match(os.path.basename(parts.path)) + if found: + return ("flickr", found.group(1)) + + page = urllib.parse.urlparse(node.get("page") or "") + if page.netloc.endswith("flickr.com"): + found = FLICKR_PAGE.search(page.path) + if found: + return ("flickr", found.group(1)) + if page.netloc == "flic.kr": + found = FLICKR_SHORT.match(page.path) + if found: + return ("flickr", str(base58decode(found.group(1)))) + + digest = hashlib.sha1(image.encode("utf-8")).hexdigest()[:16] + return ("url", "url-" + digest) + +def groupphotos(nodes): + groups = {} + rejected = [] + for node in nodes: + if not node["image"]: + continue + key = photokey(node) + if key is None: + rejected.append((node, "node_image is not a url: %r" % node["image"])) + continue + source, ident = key + group = groups.get(ident) + if group is None: + group = groups[ident] = { + "key": ident, "source": source, + "image": node["image"], "page": node["page"], + "copyright": "", "licence": "", "nodes": [], + } + if source == "flickr": + group["id"] = ident + group["nodes"].append(node) + for field in ("copyright", "licence", "page"): + if node[field] and not group[field]: + group[field] = node[field] + return (groups, rejected) + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index 28d1773..f49b068 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -105,6 +105,132 @@ class TestCase(unittest.TestCase): return path +def index_entry(param, image, page="", copyright="", licence="", lat="51.5", long="-0.1", name=None): + md = {} + if image: + md["node_image"] = [image] + if page: + md["node_image_url"] = [page] + if copyright: + md["node_image_copyright"] = [copyright] + if licence: + md["node_image_licence"] = [licence] + return {"param": param, "name": name or param.replace("_", " "), + "wgs84_lat": lat, "wgs84_long": long, "has_geodata": 1, + "node_data": {"metadata": md}} + + +class Index(TestCase): + def test_readindex_tolerates_invalid_utf8(self): + entry = index_entry("Café,_N1", "https://farm1.static.flickr.com/1/123_abc_m.jpg") + raw = json.dumps([entry], ensure_ascii=False).encode("utf-8").replace("\xe9".encode(), b"\xc2") + nodes = rb.readindex(raw) + self.assertEqual(len(nodes), 1) + self.assertIn("�", nodes[0]["param"]) + + def test_readindex_flattens_fields(self): + raw = json.dumps([index_entry("Pub,_E1", "https://farm1.static.flickr.com/1/123_abc_m.jpg", + page="https://www.flickr.com/photos/x/123/", + copyright="Someone", licence="https://creativecommons.org/licenses/by-sa/2.0/"), + index_entry("Nopic,_E2", "")]).encode() + nodes = rb.readindex(raw) + self.assertEqual(len(nodes), 2) + n = nodes[0] + self.assertEqual(n["param"], "Pub,_E1") + self.assertEqual(n["name"], "Pub, E1") + self.assertEqual(n["image"], "https://farm1.static.flickr.com/1/123_abc_m.jpg") + self.assertEqual(n["page"], "https://www.flickr.com/photos/x/123/") + self.assertEqual(n["copyright"], "Someone") + self.assertEqual(n["licence"], "https://creativecommons.org/licenses/by-sa/2.0/") + self.assertEqual((n["lat"], n["long"]), ("51.5", "-0.1")) + self.assertEqual(n["url"], "https://london.randomness.org.uk/wiki.cgi?Pub%2C_E1") + self.assertEqual(nodes[1]["image"], "") + + def test_zero_coordinates_are_kept(self): + raw = json.dumps([index_entry("Null,_Island", "https://x/1_a_m.jpg", lat=0, long=0)]).encode() + node = rb.readindex(raw)[0] + self.assertEqual((node["lat"], node["long"]), ("0", "0")) + + def test_missing_coordinates_are_empty(self): + raw = json.dumps([index_entry("Nowhere", "https://x/1_a_m.jpg", lat=None, long=None)]).encode() + node = rb.readindex(raw)[0] + self.assertEqual((node["lat"], node["long"]), ("", "")) + + def test_loadindex_fetches(self): + rb.OPEN = FakeOpen(FakeResponse(json.dumps([index_entry("A", "")]).encode())) + self.assertEqual(len(rb.loadindex("http://t/index")), 1) + + +class PhotoKeys(TestCase): + def key(self, image, page=""): + return rb.photokey({"image": image, "page": page, "param": "X"}) + + def test_id_from_thumbnail_filename(self): + self.assertEqual(self.key("https://farm4.static.flickr.com/3188/3002966579_97596aea2e_m.jpg"), + ("flickr", "3002966579")) + self.assertEqual(self.key("https://live.staticflickr.com/65535/52123_abc_m.jpg"), + ("flickr", "52123")) + + def test_id_from_page_when_thumbnail_is_odd(self): + self.assertEqual(self.key("https://www.flickr.com/some/other.jpg", + "https://www.flickr.com/photos/kake/3002966579/in/set-1/"), + ("flickr", "3002966579")) + self.assertEqual(self.key("https://c1.staticflickr.com/x.jpg", + "https://secure.flickr.com/photos/12345678@N00/98765"), + ("flickr", "98765")) + + def test_short_link_is_base58(self): + self.assertEqual(rb.base58decode("5zmZkR"), 3002966579) + self.assertEqual(self.key("https://c1.staticflickr.com/x.jpg", "https://flic.kr/p/5zmZkR"), + ("flickr", "3002966579")) + + def test_non_flickr_hosts_key_on_url_hash(self): + source, key = self.key("https://www.geograph.org.uk/photos/12/34/1234.jpg", + "https://www.geograph.org.uk/photo/1234") + self.assertEqual(source, "url") + self.assertRegex(key, r"^url-[0-9a-f]{16}$") + + def test_flickr_cdn_without_id_falls_back_to_url_key(self): + source, key = self.key("https://c1.staticflickr.com/nodigits.jpg") + self.assertEqual(source, "url") + + def test_not_a_url_is_none(self): + self.assertIsNone(self.key("not a url")) + self.assertIsNone(self.key("")) + + +class Grouping(TestCase): + def test_nodes_sharing_a_photo_group_together(self): + nodes = rb.readindex(json.dumps([ + index_entry("A,_E1", "https://farm1.static.flickr.com/1/111_a_m.jpg", + page="https://www.flickr.com/photos/x/111/", copyright="P", licence="L"), + index_entry("B,_E2", "https://farm1.static.flickr.com/1/111_a_m.jpg", + page="https://www.flickr.com/photos/x/111/"), + index_entry("C,_E3", "https://farm1.static.flickr.com/1/222_b_m.jpg"), + index_entry("D,_E4", ""), + index_entry("E,_E5", "junk"), + ]).encode()) + groups, rejected = rb.groupphotos(nodes) + self.assertEqual(sorted(groups), ["111", "222"]) + g = groups["111"] + self.assertEqual(g["source"], "flickr") + self.assertEqual(g["id"], "111") + self.assertEqual([n["param"] for n in g["nodes"]], ["A,_E1", "B,_E2"]) + self.assertEqual(g["copyright"], "P") + self.assertEqual(g["licence"], "L") + self.assertEqual(g["page"], "https://www.flickr.com/photos/x/111/") + self.assertEqual(len(rejected), 1) + self.assertEqual(rejected[0][0]["param"], "E,_E5") + + def test_first_non_empty_copyright_wins(self): + nodes = rb.readindex(json.dumps([ + index_entry("A", "https://farm1.static.flickr.com/1/111_a_m.jpg"), + index_entry("B", "https://farm1.static.flickr.com/1/111_a_m.jpg", copyright="Later"), + ]).encode()) + groups, _ = rb.groupphotos(nodes) + self.assertEqual(groups["111"]["copyright"], "Later") + + class Output(TestCase): def test_say_prints_unless_quiet(self): rb.QUIET = False