diff options
| -rw-r--r-- | .forgejo/workflows/mirror-sync.yml | 69 | ||||
| -rw-r--r-- | AGENTS.md | 45 | ||||
| -rw-r--r-- | README.md | 112 | ||||
| -rw-r--r-- | deploy/config.json | 88 | ||||
| -rw-r--r-- | deploy/docker-compose.yml | 28 | ||||
| -rw-r--r-- | deploy/nginx-edge-img.conf | 68 | ||||
| -rwxr-xr-x | scripts/verify-mirror.sh | 191 |
7 files changed, 601 insertions, 0 deletions
diff --git a/.forgejo/workflows/mirror-sync.yml b/.forgejo/workflows/mirror-sync.yml new file mode 100644 index 0000000..a95bad4 --- /dev/null +++ b/.forgejo/workflows/mirror-sync.yml @@ -0,0 +1,69 @@ +name: Mirror distroless images + +# Keeps https://img.dieschoenewolke.de in step with gcr.io/distroless. +# +# The copying is done by zot's sync extension on the VPS, not by this job: the +# registry diffs digests itself, pulls only what changed, and can do so while +# the home Forgejo instance (and this runner) is down. This workflow is the +# auditor — it compares upstream and mirror digests, pull-verifies anything new, +# and only then writes a `synced-<repo>-<tag>-<digest12>` marker tag. +# +# That is the same "marker last" discipline as the Pureblog release pipeline: +# a marker exists only for a ref that was proven pullable by digest. A run that +# dies before the push leaves no marker, so the next run retries. +# +# gcr.io load: one anonymous token request per repo plus one manifest request +# per ref = 8 requests per run, every 6 hours. No per-request storm, no docker +# login, nothing that could hit an anonymous rate limit. + +on: + schedule: + - cron: "17 */6 * * *" + workflow_dispatch: + inputs: + verify_all: + description: "Pull-verify every ref, not just new digests (1/0)" + required: false + default: "0" + type: string + +concurrency: + group: distroless-mirror-sync + # Never cancel a run half way: a cancelled run could leave a tag pushed + # without the verification it claims. Runs are minutes apart anyway. + cancel-in-progress: false + +jobs: + verify: + runs-on: docker-cli + timeout-minutes: 30 + container: + image: docker:29-cli + steps: + - name: Install workflow tools + run: apk add --no-cache bash curl git jq + + - name: Check out the marker tags + uses: https://github.com/actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Compare upstream and mirror, verify, mark + env: + # docker:cli keeps the CLI in /usr/local/bin; the runner's DOCKER_HOST + # points at the daemon, so the pull goes over the public hostname and + # the real TLS/edge path. + VERIFY_PULL: ${{ inputs.verify_all == '1' && 'always' || 'new' }} + MARKERS_FILE: ${{ runner.temp }}/mirror-markers + run: ./scripts/verify-mirror.sh + + - name: Publish marker tags + run: | + set -eu + file="${MARKERS_FILE:-$RUNNER_TEMP/mirror-markers}" + [ -s "$file" ] || { echo "no new markers"; exit 0; } + while read -r marker; do + [ -n "$marker" ] || continue + git push origin "refs/tags/$marker" + echo "marked $marker" + done <"$file" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..594ffaa --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,45 @@ +# AGENTS — distroless-mirror + +Public read-only OCI mirror of `gcr.io/distroless/{static,base,cc,python3}` on +`img.dieschoenewolke.de`, served by zot on the VPS. `README.md` has the +architecture, the deploy sequence and the zot-vs-registry:2 rationale. + +## Rules + +- **Read-only is a property of the registry, not of the edge.** Keep + `http.accessControl.repositories."**".anonymousPolicy` at `["read"]` and keep + the config free of any `auth` block. Adding htpasswd or an LDAP block would + create an authenticated write path that the internet could reach. +- The port publish stays `127.0.0.1:5000`. The edge vhost is the only entry + point; never publish zot on `0.0.0.0` or a host interface. +- Keep the tag allowlist short (`latest|nonroot|debug|debug-nonroot`). It bounds + both disk and what an anonymous client can make zot fetch from gcr.io. +- `platforms` stays `linux/amd64` + `linux/arm64`. Without the allowlist zot + copies every child of the index (arm, ppc64le, riscv64, s390x) and the mirror + triples in size on a 28 GB root. +- Marker tags (`synced-<repo>-<tag>-<digest12>`) are written only after a + successful pull **by digest** through the public hostname. Do not move tag + creation before the verification, and do not mark from the digest comparison + alone. +- Sync one way only: VPS pulls from gcr.io; nothing pushes into the registry. + +## Traps + +- `edge-nginx/nginx/default.conf` is a single-file bind mount. Write in place + (`cat new > default.conf`), then `docker exec edge-nginx nginx -t` and + `nginx -s reload` (or force-recreate). Replacing the file breaks the mount. +- AppSec must stay off for this host: docker/containerd/skopeo/crane user + agents trip CRS scanner rules. The CrowdSec IP bouncer still applies. +- Do not test from the VPS: its own requests go through AppSec and can be + challenged. Test from codeserver over the public name. +- The wildcard `*.dieschoenewolke.de` SAN covers `img`; a new **name** needs a + DNS record, not a certificate change. A new apex-level domain would need the + certbot SAN list and a 300 s deSEC propagation wait. +- zot's periodic sync updates an existing tag only on the next `pollInterval` + (2 h). A digest mismatch in the auditor is therefore "pending", not "failed"; + only a 404 for a ref that was already marked is a regression. +- `proxy_buffering off` and `proxy_max_temp_file_size 0` are deliberate: layer + blobs are streamed, not spooled onto the 28 GB root filesystem. +- zot GC removes untagged manifests after `gcDelay` (1 h). Never rely on an + untagged digest staying pullable — keep the tag (or a marker tag) that + references it. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f9d5a60 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# distroless-mirror + +Public, read-only OCI mirror of the four Google distroless images, served from +the VPS on <https://img.dieschoenewolke.de>: + +| Pull this | Equals upstream | +| --- | --- | +| `img.dieschoenewolke.de/distroless/static:<tag>` | `gcr.io/distroless/static:<tag>` | +| `img.dieschoenewolke.de/distroless/base:<tag>` | `gcr.io/distroless/base:<tag>` | +| `img.dieschoenewolke.de/distroless/cc:<tag>` | `gcr.io/distroless/cc:<tag>` | +| `img.dieschoenewolke.de/distroless/python3:<tag>` | `gcr.io/distroless/python3:<tag>` | + +`<tag>` is one of `latest`, `nonroot`, `debug`, `debug-nonroot` — the four +multi-arch tags upstream publishes for every repo. Digest-identical mirrors of +the upstream index; `linux/amd64` and `linux/arm64` children are mirrored, the +rare architectures (arm, ppc64le, riscv64, s390x) are not. + +Why: the home Forgejo registry is VPN-only, so nothing public could pull these +images. Like the cgit mirror (`git.dieschoenewolke.de`) this one lives on the +VPS and keeps serving while the home instance is down. + +``` +docker pull img.dieschoenewolke.de/distroless/static:nonroot +docker pull img.dieschoenewolke.de/distroless/python3@sha256:<index digest> +incus remote add img https://img.dieschoenewolke.de --protocol oci +``` + +## Layout + +| Path | Purpose | +| --- | --- | +| `deploy/docker-compose.yml` | VPS stack (`/root/Compose/Webserver/Registry`), zot bound to `127.0.0.1:5000` | +| `deploy/config.json` | zot config: anonymous read-only, sync from `gcr.io` | +| `deploy/nginx-edge-img.conf` | Server block for `edge-nginx` | +| `scripts/verify-mirror.sh` | Digest comparison + pull verification + marker tags | +| `.forgejo/workflows/mirror-sync.yml` | Scheduled auditor (every 6 h) | + +## Why zot and not `distribution/registry:2` + CI `skopeo copy` + +Both were on the table. `registry:2` would have made the Forgejo workflow the +only writer (`skopeo copy --all` per changed digest) and would have needed a +push credential plus an edge carve-out for that credential, or a second +VPN-only push port. + +zot wins on three points that matter here: + +1. **No push surface at all.** With no `auth` block and + `anonymousPolicy: ["read"]`, zot has no path for an authenticated write — + not from the internet, not from the LAN, not from a leaked token. With + `registry:2`, read-only can only be enforced by the edge (`limit_except GET`) + plus a separate authenticated push route; the write capability exists in the + process. +2. **Freshness does not depend on the home host.** zot pulls from gcr.io on the + VPS. A `registry:2` + CI design stops updating whenever the home Forgejo (or + its runner) is down, because the copy job runs there. +3. **`dedupe`.** The four repos share most layers; zot hardlinks identical + blobs instead of storing them per repository. `registry:2` stores blobs + globally too, but has no tag-level `dedupe` and no built-in upstream sync, so + a stale mirror is a CI bug rather than a config value. + +The cost of that choice: zot is a second non-standard registry binary to keep +updated, and the tag allowlist lives in `deploy/config.json` rather than in the +workflow. The auditor workflow covers the second half by failing when a ref we +already marked disappears. + +## Deploy + +```sh +ssh root@vpn +mkdir -p /root/Compose/Webserver/Registry +# copy deploy/docker-compose.yml, deploy/config.json +cd /root/Compose/Webserver/Registry && docker compose up -d +``` + +Edge: merge `deploy/nginx-edge-img.conf` into +`/root/Compose/Webserver/NginxEdge/nginx/default.conf` **in place** +(`cat new > default.conf` — it is a single-file bind mount), then + +```sh +docker exec edge-nginx nginx -t +docker exec edge-nginx nginx -s reload +``` + +Rollback: remove the block again (`git checkout`/`cp` the backup, write in +place, reload) and `docker compose -p registry down` in the Registry directory. +The `*.dieschoenewolke.de` wildcard certificate already covers `img`, so no +certificate change is involved. + +DNS: `A 45.142.177.86` and `AAAA 2a03:4000:4a:2ba::1` for +`img.dieschoenewolke.de` (deSEC). + +## Verify + +```sh +curl -sI https://img.dieschoenewolke.de/v2/ # 200 +docker pull img.dieschoenewolke.de/distroless/static:nonroot +docker run --rm img.dieschoenewolke.de/distroless/python3:nonroot -V +./scripts/verify-mirror.sh # digest comparison + markers +``` + +From the VPS itself the request hits CrowdSec AppSec and can be challenged — +test from another host. + +## Notes + +- The edge disables AppSec for this host (docker/skopeo user agents trip CRS + bot rules); the CrowdSec IP bouncer still applies and `limit_except GET` + rejects every write verb at the edge. +- `distribution/registry`-style `/v2/` on `git.dieschoenewolke.de` is not + usable: OCI clients derive `<host>/<repo>` from the reference, so the same + hostname would need a path-stripping vhost and `GET /v2/` would collide with + cgit. Hence the dedicated name. diff --git a/deploy/config.json b/deploy/config.json new file mode 100644 index 0000000..642046a --- /dev/null +++ b/deploy/config.json @@ -0,0 +1,88 @@ +{ + "distSpecVersion": "1.1.1", + "storage": { + "rootDirectory": "/var/lib/registry", + "gc": true, + "gcDelay": "1h", + "dedupe": true + }, + "http": { + "address": "0.0.0.0", + "port": "5000", + "realm": "distroless-mirror", + "accessControl": { + "repositories": { + "**": { + "anonymousPolicy": [ + "read" + ] + } + } + } + }, + "log": { + "level": "info", + "output": "/dev/stdout" + }, + "extensions": { + "sync": { + "enable": true, + "downloadDir": "/var/lib/registry/.sync", + "registries": [ + { + "urls": [ + "https://gcr.io" + ], + "tlsVerify": true, + "onDemand": true, + "manifestCheckInterval": "30m", + "maxRetries": 3, + "retryDelay": "5m", + "maxRetryDelay": "30m", + "content": [ + { + "prefix": "distroless/static", + "tags": { + "regex": "^(latest|nonroot|debug|debug-nonroot)$" + }, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + }, + { + "prefix": "distroless/base", + "tags": { + "regex": "^(latest|nonroot|debug|debug-nonroot)$" + }, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + }, + { + "prefix": "distroless/cc", + "tags": { + "regex": "^(latest|nonroot|debug|debug-nonroot)$" + }, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + }, + { + "prefix": "distroless/python3", + "tags": { + "regex": "^(latest|nonroot|debug|debug-nonroot)$" + }, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + } + ] + } + ] + } + } +} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..e486a2f --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,28 @@ +# Public read-only OCI mirror of the four upstream Google distroless images. +# +# Serving path: public :443 -> edge-nginx (host network, TLS + CrowdSec) -> +# 127.0.0.1:5000 -> this container. The port publish is loopback-only on +# purpose: nothing on any host interface can reach zot directly, so the edge +# vhost is the only entry point. +# +# There is no `auth` block and anonymous users only get `read`, so the registry +# has no push surface at all - not from the internet and not from the LAN. +# Content arrives exclusively through the built-in sync extension pulling from +# gcr.io, which is also why the mirror keeps working while the home Forgejo +# instance (and its runner) is down. +services: + zot: + image: ghcr.io/project-zot/zot:v2.1.23 + container_name: zot-mirror + command: ["serve", "/etc/zot/config.json"] + restart: unless-stopped + ports: + - "127.0.0.1:5000:5000" + volumes: + - ./data:/var/lib/registry + - ./config.json:/etc/zot/config.json:ro + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/deploy/nginx-edge-img.conf b/deploy/nginx-edge-img.conf new file mode 100644 index 0000000..3d6ac16 --- /dev/null +++ b/deploy/nginx-edge-img.conf @@ -0,0 +1,68 @@ +# --- img.dieschoenewolke.de: public read-only OCI mirror --------------------- +# Merge this block into /root/Compose/Webserver/NginxEdge/nginx/default.conf +# (write in place: `cat new > default.conf`) and force-recreate edge-nginx. +# +# docker/containerd/skopeo/crane all speak plain GET+HEAD with a Go or Docker +# user agent. CRS treats those as scanners, so AppSec is off for this host - the +# CrowdSec IP bouncer still applies and the registry itself is read-only. +# `limit_except GET` is the second lock: even if zot were reconfigured, the edge +# refuses every write verb. +# +# A path-based /v2/ on git.dieschoenewolke.de is NOT an option: OCI clients +# build the repository name from <host>/<path>, so a shared host would force +# names like git.dieschoenewolke.de/distroless/static and the /v2/ base path +# would have to be stripped by the edge. docker's ping is +# GET /v2/ (not /git/v2/), so the mirror gets its own name. The wildcard +# *.dieschoenewolke.de certificate already covers it - no cert change. +server { + listen 443 ssl; + listen [::]:443 ssl; + listen 443 quic; + listen [::]:443 quic; + http2 on; + server_name img.dieschoenewolke.de; + access_log /dev/stdout edge; + + set $crowdsec_disable_appsec 1; + + add_header Alt-Svc 'h3=":443"; ma=86400' always; + + ssl_certificate /certs/edge.crt; + ssl_certificate_key /certs/edge.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_ecdh_curve X25519:prime256v1:secp384r1; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_session_tickets off; + + more_set_headers + "Strict-Transport-Security: max-age=31536000; includeSubDomains" + "X-Content-Type-Options: nosniff" + "Referrer-Policy: strict-origin-when-cross-origin"; + + # a manifest/index is a few hundred KB; layers are streamed, never buffered + client_max_body_size 4m; + + location / { + limit_except GET { + deny all; + } + + proxy_pass http://127.0.0.1:5000; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + # stream blobs instead of spooling them to the edge disk (28 GB root) + proxy_buffering off; + proxy_max_temp_file_size 0; + proxy_request_buffering off; + proxy_read_timeout 900s; + proxy_send_timeout 900s; + proxy_cache off; + } +} diff --git a/scripts/verify-mirror.sh b/scripts/verify-mirror.sh new file mode 100755 index 0000000..42f10bf --- /dev/null +++ b/scripts/verify-mirror.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# verify-mirror.sh — prove that the public mirror serves exactly what upstream +# serves, and mark every verified ref with a git tag. +# +# For each repo/tag pair it compares the *index* digest advertised by gcr.io +# with the digest advertised by the mirror. Only when both agree does it pull +# the image by digest through the public hostname (the real client path) and +# create the marker tag `synced-<repo>-<tag>-<digest12>`. +# +# Marker tags are created locally; the workflow pushes them. Writing the marker +# last is the whole point: its existence means "verified pullable by digest", +# not merely "upstream has a newer tag". +# +# Exit status: +# 0 every ref resolved; new refs were verified and marked (or none changed) +# 1 the mirror answered with an error, or upstream lost a ref we mirror +# 2 usage / configuration error +# +# Environment: +# MIRROR public registry host (default img.dieschoenewolke.de) +# UPSTREAM upstream registry host (default gcr.io) +# REPOS space separated repo names under distroless/ (default the four) +# TAGS space separated tags (default the four upstream variants) +# VERIFY_PULL new (default) | always | never +# PLATFORM platform used for the verification pull (default linux/amd64) + +set -uo pipefail + +MIRROR="${MIRROR:-img.dieschoenewolke.de}" +UPSTREAM="${UPSTREAM:-gcr.io}" +REPOS="${REPOS:-static base cc python3}" +TAGS="${TAGS:-latest nonroot debug debug-nonroot}" +VERIFY_PULL="${VERIFY_PULL:-new}" +PLATFORM="${PLATFORM:-linux/amd64}" + +ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json' + +say() { printf '%s\n' "$*"; } +warn() { printf '%s\n' "$*" >&2; } + +case "$VERIFY_PULL" in + new | always | never) ;; + *) + warn "VERIFY_PULL must be new, always or never" + exit 2 + ;; +esac + +command -v curl >/dev/null || { + warn "curl is required" + exit 2 +} +command -v jq >/dev/null || { + warn "jq is required" + exit 2 +} +if [ "$VERIFY_PULL" != "never" ]; then + command -v docker >/dev/null || { + warn "docker is required unless VERIFY_PULL=never" + exit 2 + } +fi + +# Ask <url> for its manifest. Sets PROBE_STATUS (HTTP code, 000 on transport +# error) and PROBE_DIGEST (empty when the registry sent no digest header). +probe() { # <url> [authorization-header] + local url=$1 auth=${2:-} headers + headers=$(mktemp) + if [ -n "$auth" ]; then + PROBE_STATUS=$(curl -sS --max-time 120 -H "$auth" -H "Accept: $ACCEPT" \ + -D "$headers" -o /dev/null -w '%{http_code}' "$url") || PROBE_STATUS=000 + else + PROBE_STATUS=$(curl -sS --max-time 120 -H "Accept: $ACCEPT" \ + -D "$headers" -o /dev/null -w '%{http_code}' "$url") || PROBE_STATUS=000 + fi + PROBE_DIGEST=$(tr -d '\r' <"$headers" \ + | awk 'tolower($1) == "docker-content-digest:" { print $2; exit }') + rm -f "$headers" +} + +upstream_token() { # <repo> + curl -fsS --max-time 60 \ + "https://$UPSTREAM/v2/token?scope=repository:distroless/$1:pull&service=$UPSTREAM" \ + | jq -r '.token // empty' +} + +new_markers="" +failures=0 +pending=0 +unchanged=0 +verified=0 + +for repo in $REPOS; do + token=$(upstream_token "$repo") || token="" + if [ -z "$token" ]; then + # Without a token we cannot compare anything, so this is a real failure. + warn "FAIL distroless/$repo: no anonymous gcr.io token" + failures=$((failures + 1)) + continue + fi + + for tag in $TAGS; do + ref="distroless/$repo:$tag" + + probe "https://$UPSTREAM/v2/distroless/$repo/manifests/$tag" "Authorization: Bearer $token" + up=$PROBE_DIGEST + if [ "$PROBE_STATUS" != "200" ] || [ -z "$up" ]; then + warn "FAIL $ref: upstream answered $PROBE_STATUS without a digest" + failures=$((failures + 1)) + continue + fi + + probe "https://$MIRROR/v2/distroless/$repo/manifests/$tag" + mirror=$PROBE_DIGEST + case "$PROBE_STATUS" in + 200) ;; + 404) + # zot has not mirrored this tag yet (new upstream tag, or a + # pull-through that failed). Nothing to mark; the next run + # retries after zot's 2h sync. + say "PENDING $ref upstream=${up#sha256:} (not mirrored yet)" + pending=$((pending + 1)) + continue + ;; + *) + warn "FAIL $ref: mirror answered $PROBE_STATUS" + failures=$((failures + 1)) + continue + ;; + esac + if [ -z "$mirror" ]; then + warn "FAIL $ref: mirror answered 200 without a digest header" + failures=$((failures + 1)) + continue + fi + + if [ "$mirror" != "$up" ]; then + # The tag moved upstream and zot's periodic sync has not caught up. + say "PENDING $ref upstream=${up#sha256:} mirror=${mirror#sha256:}" + pending=$((pending + 1)) + continue + fi + + short=${up#sha256:} + short=${short:0:12} + marker="synced-$repo-$tag-$short" + marked=0 + git rev-parse -q --verify "refs/tags/$marker" >/dev/null 2>&1 && marked=1 + + if [ "$marked" -eq 1 ] && [ "$VERIFY_PULL" != "always" ]; then + say "OK $ref ${up#sha256:}" + unchanged=$((unchanged + 1)) + continue + fi + + if [ "$VERIFY_PULL" = "never" ]; then + say "NEW $ref ${up#sha256:} (pull verification disabled)" + else + if ! docker pull --quiet --platform "$PLATFORM" "$MIRROR/$ref@$up" >/dev/null; then + warn "FAIL $ref: digest matches but the image does not pull by digest" + failures=$((failures + 1)) + continue + fi + docker image rm "$MIRROR/$ref@$up" >/dev/null 2>&1 || true + say "OK $ref ${up#sha256:} (pulled by digest)" + fi + + if [ "$marked" -eq 1 ]; then + unchanged=$((unchanged + 1)) + continue + fi + verified=$((verified + 1)) + + if ! git tag "$marker"; then + warn "FAIL $ref: could not create marker tag $marker" + failures=$((failures + 1)) + continue + fi + new_markers="$new_markers $marker" + done +done + +say "" +say "verified=$verified unchanged=$unchanged pending=$pending failures=$failures" + +if [ -n "$new_markers" ]; then + printf '%s\n' "${new_markers# }" >"${MARKERS_FILE:-/tmp/mirror-markers}" +fi + +[ "$failures" -eq 0 ] || exit 1 +exit 0 |