diff --git a/rgl-photo-backup b/rgl-photo-backup index 68d9f55..9ae6965 100755 --- a/rgl-photo-backup +++ b/rgl-photo-backup @@ -655,5 +655,129 @@ def listsidecars(directory): return found +# flickr's published licence ids +FLICKR_LICENCES = { + 0: ("All rights reserved", ""), + 1: ("CC BY-NC-SA 2.0", "https://creativecommons.org/licenses/by-nc-sa/2.0/"), + 2: ("CC BY-NC 2.0", "https://creativecommons.org/licenses/by-nc/2.0/"), + 3: ("CC BY-NC-ND 2.0", "https://creativecommons.org/licenses/by-nc-nd/2.0/"), + 4: ("CC BY 2.0", "https://creativecommons.org/licenses/by/2.0/"), + 5: ("CC BY-SA 2.0", "https://creativecommons.org/licenses/by-sa/2.0/"), + 6: ("CC BY-ND 2.0", "https://creativecommons.org/licenses/by-nd/2.0/"), + 7: ("No known copyright restrictions", "https://www.flickr.com/commons/usage/"), + 8: ("United States Government Work", "https://www.usa.gov/government-works"), + 9: ("CC0 1.0", "https://creativecommons.org/publicdomain/zero/1.0/"), + 10: ("Public Domain Mark 1.0", "https://creativecommons.org/publicdomain/mark/1.0/"), +} + +CC_PATH = re.compile(r"creativecommons\.org/(licenses/([a-z-]+)|publicdomain/(zero|mark))/(\d+\.\d+)") + +def licencelabel(url): + found = CC_PATH.search(url or "") + if not found: + return None + if found.group(2): + return "CC %s %s" % (found.group(2).upper(), found.group(4)) + if found.group(3) == "zero": + return "CC0 %s" % found.group(4) + return "Public Domain Mark %s" % found.group(4) + +def licenceslug(label): + return re.sub(r"[^a-z0-9.]+", "-", label.lower()).strip("-") + +# +# The wiki's licence url states the terms it uses the photo under, so it wins, +# named or not: quoting a url we cannot label is honest, while falling through +# to flickr's licence would state terms the wiki never claimed. Flickr's id +# fills the gap only when the wiki has nothing at all. +# +def licencefor(side): + url = (side.get("rgl") or {}).get("licence") or "" + label = licencelabel(url) + if label: + return (label, url, "rgl") + if url: + return (None, url, "wiki") + ident = (side.get("flickr") or {}).get("licence_id") + if ident in FLICKR_LICENCES: + label, url = FLICKR_LICENCES[ident] + return (label, url, "flickr") + return (None, None, None) + +def photographer(side): + name = (side.get("rgl") or {}).get("copyright") or "" + flickr = side.get("flickr") or {} + return re.sub(r"\s+", " ", name or flickr.get("owner_realname") or flickr.get("owner_name") or "").strip() + +def describeblock(side): + flickr = side.get("flickr") or {} + lines = [MARKER] + if flickr.get("title"): + lines.append("Photo: %s" % flickr["title"]) + source = flickr.get("page") or side.get("page") or "" + if source: + lines.append("Source: %s" % source) + who = photographer(side) + if who: + lines.append("Photographer: %s" % who) + label, url, origin = licencefor(side) + if label: + line = "Licence: %s %s" % (label, url) if url else "Licence: %s" % label + if origin == "flickr": + line += " (from Flickr)" + lines.append(line) + if origin == "rgl" and flickr.get("licence_id") in FLICKR_LICENCES: + flabel = FLICKR_LICENCES[flickr["licence_id"]][0] + if flabel != label: + lines.append("Note: Flickr lists this photo as %s" % flabel) + elif origin == "wiki": + lines.append("Licence: %s (from the wiki)" % url) + else: + lines.append("Licence: unknown") + lines.append("Used on:") + for node in (side.get("rgl") or {}).get("nodes") or []: + lines.append(" %s %s" % (node.get("name", ""), node.get("url", ""))) + return "\n".join(lines) + +# Descriptions cameras and old software write by themselves; these count as +# empty rather than as something worth protecting. Same list as immich-sync. +BOILERPLATE = re.compile(r"""^\s*(?: + olympus\ digital\ camera | sony\ dsc | digital\ camera | exif_?jpeg + | picasa | dcim | camera | photo | image | untitled | default + | no\ description | ola + )\s*$""", re.I | re.X) + +def isboilerplate(text): + return bool(BOILERPLATE.match(text or "")) + +# +# Where the block sits in a description: from the marker line to the end of +# the "Used on" list, which is every following line that starts with two +# spaces. Anything after that was written by someone else and is kept. +# +BLOCK = re.compile(r"(?ms)^" + re.escape(MARKER) + r"$.*?^Used on:$(?:\n [^\n]*)*") + +def mergedescription(held, block): + held = (held or "").strip() + if not held or isboilerplate(held): + return (block, "write") + found = BLOCK.search(held) + if not found: + return (held + "\n\n" + block, "append") + if found.group(0) == block: + return (held, "same") + return (held[:found.start()] + block + held[found.end():], "replace") + +def tagsfor(side): + tags = [TAG_ROOT] + label = licencefor(side)[0] + tags.append("%s/licence/%s" % (TAG_ROOT, licenceslug(label) if label else "unknown")) + who = photographer(side).replace("/", " - ") + who = re.sub(r"\s+", " ", who).strip() + if who: + tags.append("%s/photographer/%s" % (TAG_ROOT, who)) + return tags + + if __name__ == "__main__": sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py index 96850b1..177e311 100644 --- a/test_rgl_photo_backup.py +++ b/test_rgl_photo_backup.py @@ -601,3 +601,138 @@ class Sidecars(TestCase): def test_utcnow_shape(self): self.assertRegex(rb.utcnow(), r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$") + + +class Licences(TestCase): + def test_label_from_url(self): + self.assertEqual(rb.licencelabel("https://creativecommons.org/licenses/by-nc-sa/2.0/deed.en_GB"), "CC BY-NC-SA 2.0") + self.assertEqual(rb.licencelabel("https://creativecommons.org/licenses/by/2.0/"), "CC BY 2.0") + self.assertEqual(rb.licencelabel("http://creativecommons.org/publicdomain/zero/1.0/"), "CC0 1.0") + self.assertIsNone(rb.licencelabel("")) + self.assertIsNone(rb.licencelabel("https://example.com/terms")) + + def test_slug(self): + self.assertEqual(rb.licenceslug("CC BY-NC-SA 2.0"), "cc-by-nc-sa-2.0") + self.assertEqual(rb.licenceslug("All rights reserved"), "all-rights-reserved") + + def test_rgl_licence_wins(self): + side = sidecar() + self.assertEqual(rb.licencefor(side), + ("CC BY-SA 2.0", "https://creativecommons.org/licenses/by-sa/2.0/", "rgl")) + + def test_flickr_fills_a_gap(self): + side = sidecar(licence="") + label, url, origin = rb.licencefor(side) + self.assertEqual((label, origin), ("CC BY-SA 2.0", "flickr")) + self.assertIn("by-sa/2.0", url) + + def test_a_licence_url_we_cannot_name_beats_flickr(self): + side = sidecar(licence="https://example.com/our-own-terms") + self.assertEqual(rb.licencefor(side), + (None, "https://example.com/our-own-terms", "wiki")) + self.assertIn("Licence: https://example.com/our-own-terms (from the wiki)", + rb.describeblock(side)) + self.assertNotIn("from Flickr", rb.describeblock(side)) + self.assertIn("rgl/licence/unknown", rb.tagsfor(side)) + + def test_unknown_when_neither(self): + side = sidecar(licence="") + side["flickr"]["licence_id"] = None + self.assertEqual(rb.licencefor(side), (None, None, None)) + + def test_flickr_table_covers_published_ids(self): + self.assertEqual(rb.FLICKR_LICENCES[0][0], "All rights reserved") + self.assertEqual(rb.FLICKR_LICENCES[4][0], "CC BY 2.0") + self.assertEqual(rb.FLICKR_LICENCES[9][0], "CC0 1.0") + self.assertEqual(rb.FLICKR_LICENCES[10][0], "Public Domain Mark 1.0") + + +class Describing(TestCase): + def test_block_shape(self): + block = rb.describeblock(sidecar()) + self.assertEqual(block.splitlines(), [ + rb.MARKER, + "Photo: A title", + "Source: https://www.flickr.com/photos/12345678@N00/111/", + "Photographer: P", + "Licence: CC BY-SA 2.0 https://creativecommons.org/licenses/by-sa/2.0/", + "Used on:", + " A, E1 https://london.randomness.org.uk/wiki.cgi?A%2C_E1", + ]) + + def test_photographer_falls_back_to_flickr(self): + side = sidecar(copyright="") + self.assertEqual(rb.photographer(side), "Real Name") + side["flickr"]["owner_realname"] = "" + self.assertEqual(rb.photographer(side), "handle") + + def test_licence_from_flickr_is_marked_and_disagreement_noted(self): + side = sidecar(licence="") + self.assertIn("Licence: CC BY-SA 2.0 https://creativecommons.org/licenses/by-sa/2.0/ (from Flickr)", + rb.describeblock(side)) + side = sidecar(licence="https://creativecommons.org/licenses/by-nc-sa/2.0/") + block = rb.describeblock(side) + self.assertIn("Licence: CC BY-NC-SA 2.0 https://creativecommons.org/licenses/by-nc-sa/2.0/", block) + self.assertIn("Note: Flickr lists this photo as CC BY-SA 2.0", block) + + def test_no_licence_says_unknown(self): + side = sidecar(licence="") + side["flickr"]["licence_id"] = None + self.assertIn("Licence: unknown", rb.describeblock(side)) + + def test_non_flickr_photo(self): + side = sidecar(flickr=False) + block = rb.describeblock(side) + self.assertNotIn("Photo:", block) + self.assertIn("Source: https://www.geograph.org.uk/photos/1.jpg", block) + + def test_several_pages(self): + side = sidecar(nodes=[ + {"param": "A", "name": "A", "url": "https://x/A", "lat": "", "long": ""}, + {"param": "B", "name": "B", "url": "https://x/B", "lat": "", "long": ""}]) + self.assertTrue(rb.describeblock(side).endswith("Used on:\n A https://x/A\n B https://x/B")) + + +class Merging(TestCase): + block = "%s\nPhoto: x\nUsed on:\n A https://x/A" % rb.MARKER + + def test_empty_or_boilerplate_is_written(self): + self.assertEqual(rb.mergedescription("", self.block), (self.block, "write")) + self.assertEqual(rb.mergedescription("OLYMPUS DIGITAL CAMERA", self.block), (self.block, "write")) + + def test_existing_text_is_kept_above(self): + text, action = rb.mergedescription("My caption", self.block) + self.assertEqual(action, "append") + self.assertEqual(text, "My caption\n\n" + self.block) + + def test_same_block_is_quiet(self): + held = "My caption\n\n" + self.block + self.assertEqual(rb.mergedescription(held, self.block), (held, "same")) + + def test_changed_block_is_replaced_in_place(self): + held = "My caption\n\n" + self.block + "\n\nTrailing note" + newblock = self.block + "\n B https://x/B" + text, action = rb.mergedescription(held, newblock) + self.assertEqual(action, "replace") + self.assertEqual(text, "My caption\n\n" + newblock + "\n\nTrailing note") + + def test_isboilerplate(self): + self.assertTrue(rb.isboilerplate("SONY DSC")) + self.assertFalse(rb.isboilerplate("The 100 Club")) + + +class Tags(TestCase): + def test_tags(self): + self.assertEqual(rb.tagsfor(sidecar()), + ["rgl", "rgl/licence/cc-by-sa-2.0", "rgl/photographer/P"]) + + def test_unknown_licence_and_blank_photographer(self): + side = sidecar(licence="", copyright="") + side["flickr"]["licence_id"] = None + side["flickr"]["owner_realname"] = "" + side["flickr"]["owner_name"] = "" + self.assertEqual(rb.tagsfor(side), ["rgl", "rgl/licence/unknown"]) + + def test_photographer_whitespace_and_slashes(self): + side = sidecar(copyright=" Some / One ") + self.assertIn("rgl/photographer/Some - One", rb.tagsfor(side))