bob/rgl-photo-backup
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
fc434a3d9bf9e18793c72b67636bfc719639e114
rgl-photo-backup/rgl-photo-backup
T
bobandClaude Fable 5.1 fc434a3d9b Ask flickr for each photo's details and its largest size 
Two unsigned json calls per photo: getInfo for title, owner, dates and
licence, getSizes for the renditions. Sizes come back ordered best-first so
a missing original can fall through to the next. Not-found and permission
errors become PhotoGone so the run can name the wiki pages affected.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-10 12:07:44 +01:00

605 lines
23 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
#
# Back up the photos the Randomness Guide to London shows on its pages.
#
# Phase one reads the wiki's index, asks flickr for the largest size of each
# photo and writes it to a backup directory beside a JSON sidecar of what the
# wiki and flickr know about it. Phase two mirrors that directory into an
# immich album, carrying the source, licence, photographer and the pages that
# use it into the description, tags, capture date and location.
#
import argparse
import base64
import datetime
import email.utils
import hashlib
import json
import os
import random
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
INDEX_URL = "https://london.randomness.org.uk/wiki.cgi?action=index;format=json"
NODE_URL = "https://london.randomness.org.uk/wiki.cgi?%s"
REST_URL = "https://api.flickr.com/services/rest/"
FLICKR_CONFIG = "~/.config/flickrtouchr/auth.yml"
IMMICH_CONFIG = "~/.config/immich/auth.yml"
SETTINGS_FILE = "~/.config/rgl-photo-backup/settings.yml"
MARKER = "Randomness Guide to London photo"
ALBUM = "RGL"
TAG_ROOT = "rgl"
DEVICE_ID = "rgl-photo-backup"
HASHES = ".immich-hashes.json"
PAGE = 1000 # immich search page
BATCH = 500 # ids per album/tag write
CHUNK = 1024 * 1024 # hashing
DOWNLOAD_CHUNK = 256 * 1024
DOWNLOAD_CAP = 100 * 1024 * 1024 # no wiki photo is this big; something is wrong
BACKOFF_CAP = 300.0
ENV = "RGL_PHOTO_BACKUP_"
# Every default in one place. applysettings() moves the constants below, so a
# default read from one of them would drift with the run it configured.
DEFAULTS = {
"flickr_auth": FLICKR_CONFIG,
"immich_auth": IMMICH_CONFIG,
"index_url": INDEX_URL,
"fetch": True,
"sync": True,
"dry_run": False,
"limit": None,
"pace": 1.0,
"quiet": False,
"attempts": 6,
"backoff": 5.0,
"timeout": 60.0,
"user_agent": "rgl-photo-backup/1.0",
"debug": False,
}
HTTP_TIMEOUT = float(os.environ.get(ENV + "TIMEOUT", DEFAULTS["timeout"]))
USER_AGENT = os.environ.get(ENV + "USER_AGENT", DEFAULTS["user_agent"])
ATTEMPTS = int(os.environ.get(ENV + "ATTEMPTS", DEFAULTS["attempts"]))
BACKOFF_BASE = float(os.environ.get(ENV + "BACKOFF", DEFAULTS["backoff"]))
PACE = float(os.environ.get(ENV + "PACE", DEFAULTS["pace"]))
DEBUG = bool(os.environ.get(ENV + "DEBUG"))
QUIET = False
# Replaced by tests so nothing touches the network or sleeps
OPEN = urllib.request.urlopen
SLEEP = time.sleep
STATS = {"fetched": 0, "skipped": 0, "rewritten": 0, "uploaded": 0,
"existing": 0, "updated": 0, "failed": 0, "retries": 0}
class BackupError(Exception):
pass
class PhotoGone(BackupError):
"""flickr no longer serves this photo: deleted, or made private."""
class HTTPFailure(BackupError):
def __init__(self, message, code=None):
super().__init__(message)
self.code = code
def say(msg=""):
if not QUIET:
print(msg)
def warn(msg):
print(msg, file=sys.stderr)
def error(msg):
print("error: %s" % msg, file=sys.stderr)
def debug(msg):
if DEBUG:
print("debug: %s" % msg, file=sys.stderr)
#
# All three config files are flat "name: value" yaml. Parsing it properly
# would mean a dependency, so read exactly that shape. A missing file is an
# empty one.
#
def readyaml(path):
values = {}
try:
with open(os.path.expanduser(path), encoding="utf-8") as fh:
for line in fh:
if line.lstrip().startswith("#"):
continue
found = re.match(r'^\s*([A-Za-z_][\w-]*)\s*:\s*(.*?)\s*$', line)
if found:
values[found.group(1)] = found.group(2).strip().strip("\"'")
except FileNotFoundError:
pass
except OSError as e:
raise BackupError("%s: cannot be read: %s" % (path, e.strerror or e))
return values
def loadflickr(path):
key = readyaml(path).get("key")
if not key:
raise BackupError("%s: needs a flickr api 'key'" % path)
return key
def loadimmich(path):
values = readyaml(path)
url = values.get("url", "").rstrip("/")
key = values.get("key")
if not url or not key:
raise BackupError("%s: needs both a url and a key" % path)
return (url, key)
SETTING_KINDS = {
"directory": "path",
"flickr_auth": "path",
"immich_auth": "path",
"index_url": str,
"fetch": bool,
"sync": bool,
"dry_run": bool,
"limit": int,
"pace": float,
"quiet": bool,
"attempts": int,
"backoff": float,
"timeout": float,
"user_agent": str,
"debug": bool,
}
BOOLEANS = {"true": True, "yes": True, "on": True, "1": True,
"false": False, "no": False, "off": False, "0": False}
def coerce(kind, raw):
if kind == "path":
return os.path.expanduser(raw)
if kind is bool:
return BOOLEANS[raw.lower()]
return kind(raw)
def loadsettings(path):
path = os.path.expanduser(path)
settings = {}
for (name, raw) in readyaml(path).items():
kind = SETTING_KINDS.get(name)
if kind is None:
warn("%s: unknown setting '%s' ignored" % (path, name))
continue
try:
settings[name] = coerce(kind, raw)
except (KeyError, ValueError):
wanted = {bool: "true or false", int: "a whole number",
float: "a number"}.get(kind, "text")
raise BackupError("%s: '%s' must be %s" % (path, name, wanted))
return settings
def parseargs(argv=None):
parser = argparse.ArgumentParser(
description="Back up the photos the Randomness Guide to London shows "
"to a directory, then mirror them into an immich album.",
epilog="Every option can also live in the settings file; a flag beats "
"the environment, which beats the file.")
parser.add_argument("directory", nargs="?",
help="the backup directory (default: 'directory:' in "
"the settings file)")
parser.add_argument("--fetch-only", action="store_true", default=None,
help="RGL and flickr to disk, no immich")
parser.add_argument("--sync-only", action="store_true", default=None,
help="disk to immich, no RGL or flickr calls")
parser.add_argument("--dry-run", action="store_true", default=None,
help="report what would be fetched and written, change nothing")
parser.add_argument("--limit", type=int, metavar="N",
help="stop after N photos, for a trial run")
parser.add_argument("--pace", type=float, metavar="SECONDS",
help="delay between flickr calls")
parser.add_argument("--quiet", "-q", action="store_true", default=None,
help="print only failures, for cron")
parser.add_argument("--index-url", metavar="URL",
help="the wiki index endpoint (default: RGL)")
parser.add_argument("--flickr-config", metavar="FILE",
help="flickr key file (default: %s)" % FLICKR_CONFIG)
parser.add_argument("--immich-config", metavar="FILE",
help="immich url and key file (default: %s)" % IMMICH_CONFIG)
parser.add_argument("--settings", metavar="FILE", default=SETTINGS_FILE,
help="settings file (default: %s)" % SETTINGS_FILE)
parser.add_argument("--debug", action="store_true", default=None,
help="log every request to stderr")
return parser.parse_args(argv)
# Settings the environment may also supply, by key
ENVIRONMENT = {"pace": "PACE", "attempts": "ATTEMPTS", "backoff": "BACKOFF",
"timeout": "TIMEOUT", "user_agent": "USER_AGENT", "debug": "DEBUG"}
def settings(args, environ=None):
environ = os.environ if environ is None else environ
filed = loadsettings(args.settings)
def pick(name, flag=None, default=None):
if flag is not None:
return flag
env = ENVIRONMENT.get(name)
if env and environ.get(ENV + env):
try:
return coerce(SETTING_KINDS[name], environ[ENV + env])
except (KeyError, ValueError):
raise BackupError("%s%s: %r is not a valid %s"
% (ENV, env, environ[ENV + env], name))
return filed.get(name, default)
conf = {
"directory": pick("directory", args.directory),
"flickr_auth": pick("flickr_auth", args.flickr_config,
os.path.expanduser(DEFAULTS["flickr_auth"])),
"immich_auth": pick("immich_auth", args.immich_config,
os.path.expanduser(DEFAULTS["immich_auth"])),
"index_url": pick("index_url", args.index_url, DEFAULTS["index_url"]),
"fetch": pick("fetch", False if args.sync_only else None, DEFAULTS["fetch"]),
"sync": pick("sync", False if args.fetch_only else None, DEFAULTS["sync"]),
"dry_run": pick("dry_run", args.dry_run, DEFAULTS["dry_run"]),
"limit": pick("limit", args.limit, DEFAULTS["limit"]),
"pace": pick("pace", args.pace, DEFAULTS["pace"]),
"quiet": pick("quiet", args.quiet, DEFAULTS["quiet"]),
"attempts": pick("attempts", None, DEFAULTS["attempts"]),
"backoff": pick("backoff", None, DEFAULTS["backoff"]),
"timeout": pick("timeout", None, DEFAULTS["timeout"]),
"user_agent": pick("user_agent", None, DEFAULTS["user_agent"]),
"debug": pick("debug", args.debug, DEFAULTS["debug"]),
}
if conf["directory"]:
conf["directory"] = os.path.expanduser(conf["directory"])
else:
raise BackupError("no backup directory: give it on the command line, or "
"as 'directory:' in %s" % os.path.expanduser(args.settings))
if not conf["fetch"] and not conf["sync"]:
raise BackupError("--fetch-only and --sync-only together leave nothing to do")
return conf
# Flickr's key rides in the query string, and a debug log is a file or a
# cron mail that outlives the run, so keep credentials out of it.
SECRETS = re.compile(r"(?i)([?&](?:api_key|api_sig|auth_token|oauth_token)=)[^&]*")
def redact(url):
return SECRETS.sub(r"\1<redacted>", url or "")
def retryafter(e):
value = e.headers.get("Retry-After") if e.headers else None
if not value:
return None
try:
return max(0.0, float(value))
except ValueError:
pass
try:
when = email.utils.parsedate_to_datetime(value)
return max(0.0, when.timestamp() - time.time())
except Exception:
return None
# Exponential backoff with jitter, so many photos tripping a limit at once do
# not then retry in lockstep.
def backoff(attempt):
return min(BACKOFF_BASE * (2 ** (attempt - 1)), BACKOFF_CAP) * (0.5 + random.random())
#
# Fetch a url, retrying on rate limits, server errors and transient network
# failures. consume() is handed the open response so a body can be read into
# memory or streamed to disk under one retry policy. 429 and 5xx are worth
# another go; any other 4xx is our fault and will not improve. Only flickr
# needs pacing, and only flickr's limit is worth slowing the run for.
#
def retrying(url, context, consume, headers=None, data=None, method="GET", pace=False):
sent = {"User-Agent": USER_AGENT}
sent.update(headers or {})
for attempt in range(1, ATTEMPTS + 1):
if pace and PACE:
SLEEP(PACE)
debug("%s: %s %s%s" % (context, method, redact(url),
" (attempt %d)" % attempt if attempt > 1 else ""))
request = urllib.request.Request(url, data=data, method=method, headers=sent)
try:
response = OPEN(request, timeout=HTTP_TIMEOUT)
try:
return consume(response)
finally:
response.close()
except BackupError:
raise # consume() rejected the body; retrying will not help
except urllib.error.HTTPError as e:
if e.code != 429 and e.code < 500:
detail = ""
try:
detail = ": " + e.read().decode("utf-8", "replace").strip()[:200]
except Exception:
pass
e.close()
raise HTTPFailure("%s: HTTP %s %s%s" % (context, e.code, e.reason, detail),
code=e.code)
wait = retryafter(e)
reason = "HTTP %s %s" % (e.code, e.reason)
code = e.code
e.close()
except Exception as e:
wait = None
reason = "%s: %s" % (type(e).__name__, e)
code = None
if attempt == ATTEMPTS:
raise HTTPFailure("%s: giving up after %d attempts (%s)"
% (context, ATTEMPTS, reason), code=code)
if wait is None:
wait = backoff(attempt)
STATS["retries"] += 1
warn("%s: %s - retrying in %.0fs (attempt %d of %d)"
% (context, reason, wait, attempt + 1, ATTEMPTS))
SLEEP(wait)
def fetch(url, context, headers=None, data=None, method="GET", pace=False):
return retrying(url, context, lambda r: r.read(), headers, data, method, pace)
#
# Stream a url to a file, written as it arrives. Returns the Content-Type the
# server reported and the byte count. The caller passes a .part path and
# renames it once the sidecar is written too. A body past the cap is a
# redirect to something that is not a photo, or a server gone wrong; stop
# rather than fill the disk.
#
def download(url, context, path, pace=False):
def consume(response):
written = 0
with open(path, "wb") as fh:
while True:
chunk = response.read(DOWNLOAD_CHUNK)
if not chunk:
break
fh.write(chunk)
written += len(chunk)
if written > DOWNLOAD_CAP:
raise BackupError("%s: larger than %d bytes, refusing it"
% (context, DOWNLOAD_CAP))
return (response.headers.get("Content-Type", ""), written)
return retrying(url, context, consume, pace=pace)
#
# The wiki's index in json: one entry per node, metadata values wrapped in
# one-element lists. It is not valid utf-8 throughout, so decode leniently
# rather than lose the whole run to one bad byte.
#
def nodeurl(param):
return NODE_URL % urllib.parse.quote(param, safe="_.-~()!*'")
def readindex(raw):
text = raw.decode("utf-8", "replace")
try:
entries = json.loads(text)
except ValueError as e:
raise BackupError("index was not valid json: %s" % e)
nodes = []
for entry in entries:
md = (entry.get("node_data") or {}).get("metadata") or {}
def first(name):
values = md.get(name) or [""]
return (values[0] or "").strip()
def number(name): # 0 is a coordinate, not a blank
value = entry.get(name)
return "" if value is None else str(value)
param = entry.get("param") or ""
nodes.append({
"param": param,
"name": entry.get("name") or param.replace("_", " "),
"url": nodeurl(param),
"image": first("node_image"),
"page": first("node_image_url"),
"copyright": first("node_image_copyright"),
"licence": first("node_image_licence"),
"lat": number("wgs84_lat"),
"long": number("wgs84_long"),
})
return nodes
def loadindex(url):
return readindex(fetch(url, "index"))
BASE58 = "123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"
def base58decode(text):
value = 0
for ch in text:
value = value * 58 + BASE58.index(ch)
return value
FLICKR_HOSTS = re.compile(r"(^|\.)(flickr\.com|staticflickr\.com|static\.flickr\.com)$")
FLICKR_FILE = re.compile(r"^(\d+)_[0-9a-f]+(_[a-z0-9]+)?\.\w+$")
FLICKR_PAGE = re.compile(r"/photos/[^/]+/(\d+)")
FLICKR_SHORT = re.compile(r"^/p/([1-9A-HJ-NP-Za-km-z]+)")
#
# What identifies a photo: the flickr id where there is one, otherwise a hash
# of the image url. The thumbnail filename carries the id and is the most
# reliable source; the photo page is the fallback, including flic.kr links.
#
def photokey(node):
image = node.get("image") or ""
parts = urllib.parse.urlparse(image)
if parts.scheme not in ("http", "https") or not parts.netloc:
return None
if FLICKR_HOSTS.search(parts.netloc):
found = FLICKR_FILE.match(os.path.basename(parts.path))
if found:
return ("flickr", found.group(1))
page = urllib.parse.urlparse(node.get("page") or "")
if page.netloc.endswith("flickr.com"):
found = FLICKR_PAGE.search(page.path)
if found:
return ("flickr", found.group(1))
if page.netloc == "flic.kr":
found = FLICKR_SHORT.match(page.path)
if found:
return ("flickr", str(base58decode(found.group(1))))
digest = hashlib.sha1(image.encode("utf-8")).hexdigest()[:16]
return ("url", "url-" + digest)
def groupphotos(nodes):
groups = {}
rejected = []
for node in nodes:
if not node["image"]:
continue
key = photokey(node)
if key is None:
rejected.append((node, "node_image is not a url: %r" % node["image"]))
continue
source, ident = key
group = groups.get(ident)
if group is None:
group = groups[ident] = {
"key": ident, "source": source,
"image": node["image"], "page": node["page"],
"copyright": "", "licence": "", "nodes": [],
}
if source == "flickr":
group["id"] = ident
group["nodes"].append(node)
for field in ("copyright", "licence", "page"):
if node[field] and not group[field]:
group[field] = node[field]
return (groups, rejected)
#
# Flickr answers errors with HTTP 200 and stat="fail", so check the body.
# RGL's photos are public, so an api key alone is enough: no OAuth.
#
# 105 "service currently unavailable" and 106 "write operation failed" are
# flickr being busy, not this photo being wrong, and they arrive with HTTP 200
# where retrying() cannot see them, so back off and try again here.
#
FLICKR_BUSY = (105, 106)
def flickrcall(method, params, key, context=None):
context = context or method
query = dict(params)
query.update({"method": method, "api_key": key,
"format": "json", "nojsoncallback": "1"})
url = REST_URL + "?" + urllib.parse.urlencode(query)
for attempt in range(1, ATTEMPTS + 1):
body = fetch(url, context, pace=True)
try:
answer = json.loads(body.decode("utf-8", "replace"))
except ValueError:
raise BackupError("%s: response was not json: %r" % (context, body[:200]))
if answer.get("stat") == "ok":
return answer
try:
code = int(answer.get("code"))
except (TypeError, ValueError):
code = answer.get("code")
message = answer.get("message", "unknown error")
# 1 photo not found, 2 permission denied: gone as far as we can tell
if code in (1, 2):
raise PhotoGone("%s: %s" % (context, message))
if code not in FLICKR_BUSY or attempt == ATTEMPTS:
raise BackupError("%s: flickr returned error %s: %s" % (context, code, message))
wait = backoff(attempt)
STATS["retries"] += 1
warn("%s: %s - retrying in %.0fs (attempt %d of %d)"
% (context, message, wait, attempt + 1, ATTEMPTS))
SLEEP(wait)
def isotime(epoch):
try:
when = datetime.datetime.fromtimestamp(int(epoch), datetime.timezone.utc)
except (TypeError, ValueError):
return ""
return when.strftime("%Y-%m-%dT%H:%M:%SZ")
def getinfo(photoid, key):
photo = flickrcall("flickr.photos.getInfo", {"photo_id": photoid}, key,
"photo %s info" % photoid)["photo"]
owner = photo.get("owner") or {}
page = ""
for url in (photo.get("urls") or {}).get("url") or []:
if url.get("type") == "photopage":
page = url.get("_content", "")
try:
licence = int(photo.get("license"))
except (TypeError, ValueError):
licence = None
return {
"id": str(photo.get("id") or photoid),
"owner": owner.get("nsid", ""),
"owner_name": owner.get("username", ""),
"owner_realname": owner.get("realname", ""),
"title": (photo.get("title") or {}).get("_content", "").strip(),
"description": (photo.get("description") or {}).get("_content", "").strip(),
"taken": (photo.get("dates") or {}).get("taken", ""),
"posted": isotime((photo.get("dates") or {}).get("posted")),
"licence_id": licence,
"page": page,
"original_format": photo.get("originalformat", ""),
}
def getsizes(photoid, key):
answer = flickrcall("flickr.photos.getSizes", {"photo_id": photoid}, key,
"photo %s sizes" % photoid)
sizes = []
for size in (answer.get("sizes") or {}).get("size") or []:
try:
width, height = int(size.get("width")), int(size.get("height"))
except (TypeError, ValueError):
width = height = 0
sizes.append({"label": size.get("label", ""), "width": width, "height": height,
"source": size.get("source", ""), "media": size.get("media") or "photo"})
return sizes
#
# Candidates in the order to try them: the original first, then by area, so a
# 404 on one size can fall through to the next. Videos are never wiki photos.
#
def picksizes(sizes):
photos = [s for s in sizes if s["media"] == "photo" and s["source"]
and s["label"] != "Video Player"]
originals = [s for s in photos if s["label"] == "Original"]
rest = sorted((s for s in photos if s["label"] != "Original"),
key=lambda s: s["width"] * s["height"], reverse=True)
return originals + rest
IMAGE_EXTENSIONS = (".jpg", ".png", ".gif", ".tif", ".tiff", ".bmp", ".webp")
CONTENT_EXTENSIONS = {"image/jpeg": ".jpg", "image/png": ".png", "image/gif": ".gif",
"image/tiff": ".tif", "image/webp": ".webp"}
def extensionfor(source, contenttype=""):
ext = os.path.splitext(urllib.parse.urlparse(source).path)[1].lower()
if ext == ".jpeg":
return ".jpg"
if ext in IMAGE_EXTENSIONS:
return ext
kind = (contenttype or "").split(";")[0].strip().lower()
return CONTENT_EXTENSIONS.get(kind, ".jpg")
if __name__ == "__main__":
sys.exit(0)
Reference in new issue View Git Blame Copy Permalink