commit a37e3b69d02ec16872c01294d521c241c2faa970 Author: bob walker Date: Thu Sep 10 12:07:43 2026 +0100 Scaffold rgl-photo-backup with output helpers and test harness Start the script that will back up the Randomness Guide to London's photos to disk and Immich. This commit holds the constants, error classes and quiet-aware output helpers every later section relies on, plus the test loader for an extensionless script. Co-Authored-By: Claude Fable 5.1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ba427a1 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +.PHONY: test lint all + +all: lint test + +test: + python3 -m unittest -q + +lint: + python3 -m py_compile rgl-photo-backup test_rgl_photo_backup.py diff --git a/rgl-photo-backup b/rgl-photo-backup new file mode 100755 index 0000000..9826355 --- /dev/null +++ b/rgl-photo-backup @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# +# Back up the photos the Randomness Guide to London shows on its pages. +# +# Phase one reads the wiki's index, asks flickr for the largest size of each +# photo and writes it to a backup directory beside a JSON sidecar of what the +# wiki and flickr know about it. Phase two mirrors that directory into an +# immich album, carrying the source, licence, photographer and the pages that +# use it into the description, tags, capture date and location. +# + +import argparse +import base64 +import datetime +import email.utils +import hashlib +import json +import os +import random +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +INDEX_URL = "https://london.randomness.org.uk/wiki.cgi?action=index;format=json" +NODE_URL = "https://london.randomness.org.uk/wiki.cgi?%s" +REST_URL = "https://api.flickr.com/services/rest/" +FLICKR_CONFIG = "~/.config/flickrtouchr/auth.yml" +IMMICH_CONFIG = "~/.config/immich/auth.yml" +SETTINGS_FILE = "~/.config/rgl-photo-backup/settings.yml" + +MARKER = "Randomness Guide to London photo" +ALBUM = "RGL" +TAG_ROOT = "rgl" +DEVICE_ID = "rgl-photo-backup" +HASHES = ".immich-hashes.json" + +PAGE = 1000 # immich search page +BATCH = 500 # ids per album/tag write +CHUNK = 1024 * 1024 # hashing +DOWNLOAD_CHUNK = 256 * 1024 +DOWNLOAD_CAP = 100 * 1024 * 1024 # no wiki photo is this big; something is wrong +BACKOFF_CAP = 300.0 + +ENV = "RGL_PHOTO_BACKUP_" + +# Every default in one place. applysettings() moves the constants below, so a +# default read from one of them would drift with the run it configured. +DEFAULTS = { + "flickr_auth": FLICKR_CONFIG, + "immich_auth": IMMICH_CONFIG, + "index_url": INDEX_URL, + "fetch": True, + "sync": True, + "dry_run": False, + "limit": None, + "pace": 1.0, + "quiet": False, + "attempts": 6, + "backoff": 5.0, + "timeout": 60.0, + "user_agent": "rgl-photo-backup/1.0", + "debug": False, +} + +HTTP_TIMEOUT = float(os.environ.get(ENV + "TIMEOUT", DEFAULTS["timeout"])) +USER_AGENT = os.environ.get(ENV + "USER_AGENT", DEFAULTS["user_agent"]) +ATTEMPTS = int(os.environ.get(ENV + "ATTEMPTS", DEFAULTS["attempts"])) +BACKOFF_BASE = float(os.environ.get(ENV + "BACKOFF", DEFAULTS["backoff"])) +PACE = float(os.environ.get(ENV + "PACE", DEFAULTS["pace"])) +DEBUG = bool(os.environ.get(ENV + "DEBUG")) +QUIET = False + +# Replaced by tests so nothing touches the network or sleeps +OPEN = urllib.request.urlopen +SLEEP = time.sleep + +STATS = {"fetched": 0, "skipped": 0, "rewritten": 0, "uploaded": 0, + "existing": 0, "updated": 0, "failed": 0, "retries": 0} + + +class BackupError(Exception): + pass + +class PhotoGone(BackupError): + """flickr no longer serves this photo: deleted, or made private.""" + +class HTTPFailure(BackupError): + def __init__(self, message, code=None): + super().__init__(message) + self.code = code + + +def say(msg=""): + if not QUIET: + print(msg) + +def warn(msg): + print(msg, file=sys.stderr) + +def error(msg): + print("error: %s" % msg, file=sys.stderr) + +def debug(msg): + if DEBUG: + print("debug: %s" % msg, file=sys.stderr) + + +if __name__ == "__main__": + sys.exit(0) diff --git a/test_rgl_photo_backup.py b/test_rgl_photo_backup.py new file mode 100644 index 0000000..ed0b5b1 --- /dev/null +++ b/test_rgl_photo_backup.py @@ -0,0 +1,134 @@ +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import os +import re +import sys +import tempfile +import unittest +import urllib.error +import urllib.parse + +_here = os.path.dirname(os.path.abspath(__file__)) +_path = os.path.join(_here, "rgl-photo-backup") +_spec = importlib.util.spec_from_file_location( + "rgl_photo_backup", _path, + loader=importlib.machinery.SourceFileLoader("rgl_photo_backup", _path)) +rb = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(rb) + + +class FakeResponse: + """What urlopen would return: a body, a status and headers.""" + + def __init__(self, body=b"", status=200, headers=None): + self.body = body if isinstance(body, bytes) else json.dumps(body).encode() + self.status = status + self.headers = headers or {} + self._pos = 0 + + def read(self, size=-1): + if size is None or size < 0: + data, self._pos = self.body[self._pos:], len(self.body) + else: + data = self.body[self._pos:self._pos + size] + self._pos += len(data) + return data + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + pass + + +def http_error(code, body=b"", headers=None): + return urllib.error.HTTPError("http://test", code, "reason %d" % code, + headers or {}, io.BytesIO(body)) + + +class FakeOpen: + """Replaces rb.OPEN. Serves scripted responses in order and records + every request so a test can assert on method, url, headers and body.""" + + def __init__(self, *responses): + self.responses = list(responses) + self.requests = [] + + def __call__(self, request, timeout=None): + self.requests.append(request) + if not self.responses: + raise AssertionError("unexpected request: %s %s" + % (request.get_method(), request.full_url)) + answer = self.responses.pop(0) + if isinstance(answer, Exception): + raise answer + return answer + + +class TestCase(unittest.TestCase): + """Silences output and restores the module globals tests move around.""" + + GLOBALS = ("OPEN", "SLEEP", "ATTEMPTS", "BACKOFF_BASE", "PACE", "QUIET", + "DEBUG", "HTTP_TIMEOUT", "USER_AGENT", "STATS", "DOWNLOAD_CAP") + + def setUp(self): + self._output = contextlib.ExitStack() + if not os.environ.get("RGL_PHOTO_BACKUP_TEST_VERBOSE"): + self.stdout = self._output.enter_context( + contextlib.redirect_stdout(io.StringIO())) + self.stderr = self._output.enter_context( + contextlib.redirect_stderr(io.StringIO())) + self.addCleanup(self._output.close) + self._saved = {k: getattr(rb, k) for k in self.GLOBALS} + rb.SLEEP = lambda seconds: None + rb.PACE = 0.0 + rb.STATS = dict.fromkeys(rb.STATS, 0) + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dir = self.tmp.name + + def tearDown(self): + for k, v in self._saved.items(): + setattr(rb, k, v) + + def write(self, name, content): + path = os.path.join(self.dir, name) + mode = "wb" if isinstance(content, bytes) else "w" + with open(path, mode) as fh: + fh.write(content) + return path + + +class Output(TestCase): + def test_say_prints_unless_quiet(self): + rb.QUIET = False + rb.say("hello") + self.assertEqual(self.stdout.getvalue(), "hello\n") + rb.QUIET = True + rb.say("again") + self.assertEqual(self.stdout.getvalue(), "hello\n") + + def test_warn_always_reaches_stderr(self): + rb.QUIET = True + rb.warn("careful") + self.assertEqual(self.stderr.getvalue(), "careful\n") + + def test_debug_only_when_enabled(self): + rb.DEBUG = False + rb.debug("x") + self.assertEqual(self.stderr.getvalue(), "") + rb.DEBUG = True + rb.debug("y") + self.assertEqual(self.stderr.getvalue(), "debug: y\n") + + def test_errors_share_a_base(self): + self.assertTrue(issubclass(rb.PhotoGone, rb.BackupError)) + e = rb.HTTPFailure("ctx: HTTP 404", code=404) + self.assertEqual(e.code, 404) + self.assertIsInstance(e, rb.BackupError)