From 01efa47efa48decbebae198a1207df6316a964dd Mon Sep 17 00:00:00 2001 From: Joe Corall Date: Fri, 1 May 2026 11:16:08 +0000 Subject: [PATCH] [minor] expose derivative cache invalidation route --- .github/workflows/github-release.yaml | 19 ++ config.example.yaml | 10 + docs/caching.md | 40 ++- internal/config/config.go | 43 +++- internal/config/config_test.go | 94 +++++++ internal/iiif/image/v3/handler/handler.go | 236 +++++++++++++++--- .../iiif/image/v3/handler/handler_test.go | 207 ++++++++++++++- internal/server/server.go | 21 +- internal/storage/local_url.go | 25 +- internal/storage/local_url_test.go | 22 ++ internal/storage/multiplex.go | 14 ++ internal/storage/opener.go | 6 + 12 files changed, 692 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/github-release.yaml diff --git a/.github/workflows/github-release.yaml b/.github/workflows/github-release.yaml new file mode 100644 index 0000000..f489a79 --- /dev/null +++ b/.github/workflows/github-release.yaml @@ -0,0 +1,19 @@ +name: Create release + +on: + pull_request_target: + branches: + - main + types: + - closed + +jobs: + release: + if: github.event.pull_request.merged == true && !contains(github.event.pull_request.title, 'skip-release') + uses: libops/actions/.github/workflows/bump-release.yaml@ef667db8c16533a257d841e75df5c3388152b2d7 # main + with: + prefix: v + permissions: + contents: write + actions: write + secrets: inherit diff --git a/config.example.yaml b/config.example.yaml index fb80f91..c2b3c40 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -96,6 +96,16 @@ iiif: load_access: auto # Cache info.json dimensions by identifier plus source mtime/size. info_dimension_cache: true + # Optional protected route for invalidating all derivative variants for one + # identifier: POST {prefix}/{identifier}/cache/invalidate with + # Authorization: Bearer ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN}. + # If TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN is unset, + # TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN_FILE may point at a mounted secret + # file whose contents populate the token environment variable. + # Optionally restrict callers by client CIDR. When Triplet is behind a + # proxy, configure logging.trusted_proxy_cidrs so X-Forwarded-For is trusted. + # cache_invalidation_allowed_cidrs: [127.0.0.1/32, ::1/128] + # cache_invalidation_token: ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN} presentation: enabled: false prefix: /presentation/v3 diff --git a/docs/caching.md b/docs/caching.md index 1d535a8..055125d 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -18,6 +18,42 @@ cache: max_bytes: 1073741824 ``` +Derivative caches can be invalidated per identifier by configuring an image +cache invalidation token and calling the protected route: + +```yaml +iiif: + image: + cache_invalidation_token: ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN} + cache_invalidation_allowed_cidrs: + - 127.0.0.1/32 + - ::1/128 +``` + +If `TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN` is unset, +`TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN_FILE` may point at a readable mounted +secret file. Triplet reads that file before expanding the YAML configuration and +uses its contents as `TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN`. + +When `cache_invalidation_allowed_cidrs` is set, callers must match one of those +CIDRs in addition to presenting the bearer token. If Triplet runs behind a +reverse proxy, configure `logging.trusted_proxy_cidrs`; only trusted proxies are +allowed to supply the client IP via `X-Forwarded-For` or `X-Real-IP`. + +```sh +curl -X POST \ + -H "Authorization: Bearer ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN}" \ + "https://iiif.example.edu/iiif/3/https%3A%2F%2Frepo.example.edu%2Fsystem%2Ffiles%2Fimage.tif/cache/invalidate" +``` + +The route writes an invalidation marker into the derivative cache. Subsequent +image requests for that identifier use a new cache namespace, so old derivative +objects are ignored even when the source bytes and metadata have not changed. +This is useful for local URL mappings with `auth_probe: true`, where repository +permission changes may need to take effect before the configured auth-probe TTL +expires. When the source backend supports per-identifier auth caching, the same +route also clears those cached auth-probe decisions for the identifier. + ## Source cache The optional source cache stores fetched source bytes, primarily for HTTP @@ -71,10 +107,10 @@ sources: | Layer | Configuration | What is cached | Invalidation / freshness | |---|---|---|---| -| Derivative cache | `cache.root` or `cache.bucket_url`; optional `cache.max_bytes`, `cache.prefix` | Encoded IIIF image responses, keyed by identifier, source version, region, size, rotation, quality, and format. | A changed source version produces a new key. Filesystem caches can evict best-effort by size; GCS/object lifecycle is external. Failed transforms and HTTP error responses are not stored. | +| Derivative cache | `cache.root` or `cache.bucket_url`; optional `cache.max_bytes`, `cache.prefix`, `iiif.image.cache_invalidation_token` | Encoded IIIF image responses, keyed by identifier, source version, invalidation marker, region, size, rotation, quality, and format. | A changed source version produces a new key. The protected invalidation route bumps the per-identifier invalidation marker. Filesystem caches can evict best-effort by size; GCS/object lifecycle is external. Failed transforms and HTTP error responses are not stored. | | HTTP source cache | `cache.source_root` or `cache.source_bucket_url`; optional `cache.source_max_bytes`, `cache.source_prefix`, `cache.source_stale_after` | Original source bytes fetched through the HTTP source backend. | Keys are source identifiers. When `source_stale_after` is set, stale hits are served immediately and refreshed in the background. Upstream 4xx/5xx responses are not stored. | | `info.json` dimension cache | `iiif.image.info_dimension_cache` | Source dimensions used to build Image API `info.json`. | In-memory only. Entries are keyed by identifier plus source size/modtime metadata, so source changes with updated metadata miss the cache. | -| Local URL auth-probe cache | `sources.file.url_mappings[].auth_*` | Authorization probe results for local URL mappings with `auth_probe: true`. Anonymous and credentialed probes are cached separately. | In-memory only. Success, 403, and 404 results are cached for the configured TTL. Other upstream errors are not cached. 403/404 results with a `Last-Modified` newer than `auth_error_cache_min_age` are not cached. | +| Local URL auth-probe cache | `sources.file.url_mappings[].auth_*` | Authorization probe results for local URL mappings with `auth_probe: true`. Anonymous and credentialed probes are cached separately. | In-memory only. Success, 403, and 404 results are cached for the configured TTL. Other upstream errors are not cached. 403/404 results with a `Last-Modified` newer than `auth_error_cache_min_age` are not cached. The image cache invalidation route also clears matching auth-probe entries when the source backend supports it. | | libvips operation cache | `vips.cache_max_mem`, `vips.cache_max_files` | libvips in-process operation results. | Disabled by default in the example config. This is process-local and separate from Triplet's derivative/source caches. | Source caching improves performance but does not replace the HTTP source diff --git a/internal/config/config.go b/internal/config/config.go index 286f501..23e30b5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -87,6 +87,8 @@ type Image struct { Enabled bool `yaml:"enabled"` Prefix string `yaml:"prefix"` AllowedOrigins []string `yaml:"allowed_origins"` + CacheInvalidationToken string `yaml:"cache_invalidation_token"` + CacheInvalidationAllowedCIDRs []string `yaml:"cache_invalidation_allowed_cidrs"` MaxOutputPixels int64 `yaml:"max_output_pixels"` AllowUnsafeUnlimitedOutputPixels bool `yaml:"allow_unsafe_unlimited_output_pixels"` MaxSourcePixels int64 `yaml:"max_source_pixels"` @@ -203,7 +205,11 @@ func Load(path string) (*Config, error) { if err != nil { return nil, fmt.Errorf("read config %q: %w", path, err) } - expanded := os.ExpandEnv(string(b)) + env, err := configEnv() + if err != nil { + return nil, fmt.Errorf("load image cache invalidation token file: %w", err) + } + expanded := os.Expand(string(b), env) explicitUnlimitedOutput, err := explicitZeroYAMLField([]byte(expanded), "iiif", "image", "max_output_pixels") if err != nil { return nil, fmt.Errorf("parse config %q: %w", path, err) @@ -224,6 +230,36 @@ func Load(path string) (*Config, error) { return &c, nil } +func configEnv() (func(string) string, error) { + token, err := imageCacheInvalidationTokenEnv() + if err != nil { + return nil, err + } + return func(key string) string { + if key == "TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN" && token != "" { + return token + } + return os.Getenv(key) + }, nil +} + +func imageCacheInvalidationTokenEnv() (string, error) { + const tokenEnv = "TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN" + const fileEnv = tokenEnv + "_FILE" + if token := os.Getenv(tokenEnv); token != "" { + return token, nil + } + path := strings.TrimSpace(os.Getenv(fileEnv)) + if path == "" { + return "", nil + } + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s: %w", fileEnv, err) + } + return strings.TrimRight(string(b), "\r\n"), nil +} + func explicitZeroYAMLField(body []byte, path ...string) (bool, error) { var root yaml.Node if err := yaml.Unmarshal(body, &root); err != nil { @@ -385,6 +421,11 @@ func (c *Config) validate() error { if err := validateAllowedOrigins("iiif.image.allowed_origins", c.IIIF.Image.AllowedOrigins); err != nil { return err } + for _, cidr := range c.IIIF.Image.CacheInvalidationAllowedCIDRs { + if _, _, err := net.ParseCIDR(cidr); err != nil { + return fmt.Errorf("iiif.image.cache_invalidation_allowed_cidrs: invalid CIDR %q: %w", cidr, err) + } + } if !strings.HasPrefix(c.IIIF.Presentation.Prefix, "/") { return fmt.Errorf("iiif.presentation.prefix: must start with `/`, got %q", c.IIIF.Presentation.Prefix) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f97c458..1e71549 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -510,6 +510,21 @@ sources: `, wantErr: "iiif.image.allowed_origins", }, + { + name: "image cache invalidation cidr invalid", + body: ` +server: + public_base_url: http://localhost:8080 +iiif: + image: + cache_invalidation_allowed_cidrs: [not-a-cidr] +sources: + default: file + file: + root: /tmp +`, + wantErr: "iiif.image.cache_invalidation_allowed_cidrs", + }, { name: "explicit unlimited output pixels requires unsafe opt-in", body: ` @@ -546,6 +561,85 @@ sources: } } +func TestLoadImageCacheInvalidationTokenFromFile(t *testing.T) { + t.Setenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN", "") + dir := t.TempDir() + tokenPath := filepath.Join(dir, "image-cache-token") + if err := os.WriteFile(tokenPath, []byte("file-token\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN_FILE", tokenPath) + path := writeConfig(t, ` +server: + public_base_url: http://localhost:8080 +iiif: + image: + cache_invalidation_token: ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN} +sources: + default: file + file: + root: /tmp +`) + + c, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got := c.IIIF.Image.CacheInvalidationToken; got != "file-token" { + t.Fatalf("cache_invalidation_token = %q", got) + } + if got := os.Getenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN"); got != "" { + t.Fatalf("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN was mutated to %q", got) + } +} + +func TestLoadImageCacheInvalidationTokenEnvOverridesFile(t *testing.T) { + t.Setenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN", "env-token") + dir := t.TempDir() + tokenPath := filepath.Join(dir, "image-cache-token") + if err := os.WriteFile(tokenPath, []byte("file-token\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN_FILE", tokenPath) + path := writeConfig(t, ` +server: + public_base_url: http://localhost:8080 +iiif: + image: + cache_invalidation_token: ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN} +sources: + default: file + file: + root: /tmp +`) + + c, err := Load(path) + if err != nil { + t.Fatal(err) + } + if got := c.IIIF.Image.CacheInvalidationToken; got != "env-token" { + t.Fatalf("cache_invalidation_token = %q", got) + } +} + +func TestLoadImageCacheInvalidationTokenFileMissing(t *testing.T) { + t.Setenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN", "") + t.Setenv("TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN_FILE", filepath.Join(t.TempDir(), "missing")) + path := writeConfig(t, ` +server: + public_base_url: http://localhost:8080 +sources: + default: file + file: + root: /tmp +`) + + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN_FILE") { + t.Fatalf("err = %v, want token file error", err) + } +} + func TestLoadAppliesDefaults(t *testing.T) { path := writeConfig(t, ` server: diff --git a/internal/iiif/image/v3/handler/handler.go b/internal/iiif/image/v3/handler/handler.go index b601569..d40f681 100644 --- a/internal/iiif/image/v3/handler/handler.go +++ b/internal/iiif/image/v3/handler/handler.go @@ -11,11 +11,13 @@ package handler import ( "context" "crypto/sha256" + "crypto/subtle" "encoding/json" "errors" "fmt" "io" "log/slog" + "net" "net/http" "net/url" "os" @@ -36,20 +38,23 @@ import ( // Handler serves the Image API 3.0 surface mounted at Prefix. type Handler struct { - prefix string - publicBaseURL string - src storage.Opener - pipeline *pipeline.Pipeline - derivativeCache cache.Store - cors cors.Policy - infoCacheEnabled bool - infoCacheMu sync.RWMutex - infoCache map[string]cachedDimensions - infoLimits types.Limits - maxSourcePixels int64 - maxSourceBytes int64 - vipsLimiter chan struct{} - logger *slog.Logger + prefix string + publicBaseURL string + src storage.Opener + pipeline *pipeline.Pipeline + derivativeCache cache.Store + cors cors.Policy + invalidationToken string + invalidationCIDRs []*net.IPNet + trustedProxies []*net.IPNet + infoCacheEnabled bool + infoCacheMu sync.RWMutex + infoCache map[string]cachedDimensions + infoLimits types.Limits + maxSourcePixels int64 + maxSourceBytes int64 + vipsLimiter chan struct{} + logger *slog.Logger } type cachedDimensions struct { @@ -67,7 +72,7 @@ const ( // New constructs an Image API handler. // // derivCache may be nil to disable derivative caching. -func New(prefix, publicBaseURL string, src storage.Opener, pipe *pipeline.Pipeline, derivCache cache.Store, allowedOrigins []string, infoLimits types.Limits, infoCacheEnabled bool, maxSourcePixels, maxSourceBytes int64, maxConcurrentTransforms int, logger *slog.Logger) *Handler { +func New(prefix, publicBaseURL string, src storage.Opener, pipe *pipeline.Pipeline, derivCache cache.Store, allowedOrigins []string, invalidationToken string, invalidationCIDRs, trustedProxies []*net.IPNet, infoLimits types.Limits, infoCacheEnabled bool, maxSourcePixels, maxSourceBytes int64, maxConcurrentTransforms int, logger *slog.Logger) *Handler { if derivCache == nil { derivCache = cache.Noop{} } @@ -76,19 +81,22 @@ func New(prefix, publicBaseURL string, src storage.Opener, pipe *pipeline.Pipeli vipsLimiter = make(chan struct{}, maxConcurrentTransforms) } return &Handler{ - prefix: strings.TrimRight(prefix, "/"), - publicBaseURL: strings.TrimRight(publicBaseURL, "/"), - src: src, - pipeline: pipe, - derivativeCache: derivCache, - cors: cors.New(allowedOrigins, exposeHeaders), - infoCacheEnabled: infoCacheEnabled, - infoCache: map[string]cachedDimensions{}, - infoLimits: infoLimits, - maxSourcePixels: maxSourcePixels, - maxSourceBytes: maxSourceBytes, - vipsLimiter: vipsLimiter, - logger: logger, + prefix: strings.TrimRight(prefix, "/"), + publicBaseURL: strings.TrimRight(publicBaseURL, "/"), + src: src, + pipeline: pipe, + derivativeCache: derivCache, + cors: cors.New(allowedOrigins, exposeHeaders), + invalidationToken: invalidationToken, + invalidationCIDRs: invalidationCIDRs, + trustedProxies: trustedProxies, + infoCacheEnabled: infoCacheEnabled, + infoCache: map[string]cachedDimensions{}, + infoLimits: infoLimits, + maxSourcePixels: maxSourcePixels, + maxSourceBytes: maxSourceBytes, + vipsLimiter: vipsLimiter, + logger: logger, } } @@ -99,6 +107,9 @@ func (h *Handler) Register(mux *http.ServeMux) { // ServeHTTP implements http.Handler. func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if ok := h.maybeServeCacheInvalidation(w, r); ok { + return + } if r.Method != http.MethodGet && r.Method != http.MethodHead { writeError(w, http.StatusMethodNotAllowed, "method not allowed") return @@ -184,6 +195,11 @@ func (h *Handler) serveImage(w http.ResponseWriter, r *http.Request, req parse.R key, cacheable := derivativeKey(req, meta) etag := "" if cacheable { + if h.invalidationToken != "" { + if version := h.derivativeInvalidationVersion(r.Context(), req.Identifier); version != "" { + key += "#invalidate=" + version + } + } etag = derivativeETag(key) w.Header().Set("ETag", etag) if ifNoneMatchMatches(r.Header.Values("If-None-Match"), etag) { @@ -390,6 +406,166 @@ func (h *Handler) sourceMeta(ctx context.Context, identifier string) (storage.Me return meta, nil } +func (h *Handler) maybeServeCacheInvalidation(w http.ResponseWriter, r *http.Request) bool { + identifier, ok, err := h.cacheInvalidationIdentifier(r.URL.EscapedPath()) + if !ok { + return false + } + h.cors.SetHeaders(w, r) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return true + } + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method not allowed") + return true + } + if h.invalidationToken == "" { + writeError(w, http.StatusNotFound, "not found") + return true + } + if !h.authorizedInvalidation(r) { + w.Header().Set("WWW-Authenticate", `Bearer realm="triplet-image-cache"`) + writeError(w, http.StatusUnauthorized, "unauthorized") + return true + } + if !h.clientAllowedToInvalidate(r) { + writeError(w, http.StatusForbidden, "forbidden") + return true + } + version := time.Now().UTC().Format(time.RFC3339Nano) + if err := h.derivativeCache.Put(r.Context(), invalidationMarkerKey(identifier), "text/plain; charset=utf-8", strings.NewReader(version)); err != nil { + h.logger.Error("derivative cache invalidate", "identifier", redact.Identifier(identifier), "identifier_hash", redact.Hash(identifier), "err", err) + writeError(w, http.StatusInternalServerError, "failed to invalidate cache") + return true + } + h.infoCacheMu.Lock() + delete(h.infoCache, identifier) + h.infoCacheMu.Unlock() + if invalidator, ok := h.src.(storage.AuthInvalidator); ok { + if err := invalidator.InvalidateAuth(r.Context(), identifier); err != nil { + h.logger.Warn("source auth cache invalidate", "identifier", redact.Identifier(identifier), "identifier_hash", redact.Hash(identifier), "err", err) + } + } + h.logger.Info("derivative cache invalidated", "identifier", redact.Identifier(identifier), "identifier_hash", redact.Hash(identifier)) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNoContent) + return true +} + +func (h *Handler) cacheInvalidationIdentifier(path string) (string, bool, error) { + rest := strings.TrimPrefix(path, h.prefix) + if rest == path { + return "", false, nil + } + const suffix = "/cache/invalidate" + if !strings.HasSuffix(rest, suffix) { + return "", false, nil + } + raw := strings.TrimSuffix(rest, suffix) + raw = strings.TrimPrefix(raw, "/") + if raw == "" { + return "", true, fmt.Errorf("empty identifier") + } + identifier, err := url.PathUnescape(raw) + if err != nil { + return "", true, fmt.Errorf("identifier: %v", err) + } + if strings.ContainsAny(identifier, "\x00\n\r") { + return "", true, fmt.Errorf("identifier contains illegal control character") + } + return identifier, true, nil +} + +func (h *Handler) authorizedInvalidation(r *http.Request) bool { + token := bearerToken(r) + if token == "" || h.invalidationToken == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(token), []byte(h.invalidationToken)) == 1 +} + +func (h *Handler) clientAllowedToInvalidate(r *http.Request) bool { + if len(h.invalidationCIDRs) == 0 { + return true + } + ip := requestClientIP(r, h.trustedProxies) + if ip == nil { + return false + } + for _, cidr := range h.invalidationCIDRs { + if cidr.Contains(ip) { + return true + } + } + return false +} + +func bearerToken(r *http.Request) string { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { + return "" + } + return strings.TrimSpace(auth[len("Bearer "):]) +} + +func requestClientIP(r *http.Request, trustedProxies []*net.IPNet) net.IP { + remote := remoteIP(r.RemoteAddr) + if ipInCIDRs(remote, trustedProxies) { + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + ip, _, _ := strings.Cut(xff, ",") + if parsed := net.ParseIP(strings.TrimSpace(ip)); parsed != nil { + return parsed + } + } + if parsed := net.ParseIP(strings.TrimSpace(r.Header.Get("X-Real-IP"))); parsed != nil { + return parsed + } + } + return remote +} + +func remoteIP(remoteAddr string) net.IP { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + host = remoteAddr + } + return net.ParseIP(strings.TrimSpace(host)) +} + +func ipInCIDRs(ip net.IP, cidrs []*net.IPNet) bool { + if ip == nil { + return false + } + for _, cidr := range cidrs { + if cidr.Contains(ip) { + return true + } + } + return false +} + +func (h *Handler) derivativeInvalidationVersion(ctx context.Context, identifier string) string { + rc, entry, err := h.derivativeCache.Get(ctx, invalidationMarkerKey(identifier)) + if err != nil { + if !errors.Is(err, cache.ErrMiss) { + h.logger.Warn("derivative cache invalidation marker get", "identifier", redact.Identifier(identifier), "identifier_hash", redact.Hash(identifier), slog.Any("err", err)) + } + return "" + } + defer rc.Close() + b, err := io.ReadAll(io.LimitReader(rc, 256)) + if err == nil { + if version := strings.TrimSpace(string(b)); version != "" { + return version + } + } + if !entry.StoredAt.IsZero() { + return entry.StoredAt.UTC().Format(time.RFC3339Nano) + } + return "" +} + func (h *Handler) acquireVips(ctx context.Context) (func(), error) { if h.vipsLimiter == nil { return func() {}, nil @@ -485,6 +661,10 @@ func derivativeKey(req parse.Request, meta storage.Meta) (string, bool) { return key + "#source=" + version, true } +func invalidationMarkerKey(identifier string) string { + return "iiif/3/" + identifier + "/cache/invalidation" +} + func ifNoneMatchMatches(values []string, etag string) bool { for _, value := range values { for _, part := range strings.Split(value, ",") { diff --git a/internal/iiif/image/v3/handler/handler_test.go b/internal/iiif/image/v3/handler/handler_test.go index 034c581..2685a72 100644 --- a/internal/iiif/image/v3/handler/handler_test.go +++ b/internal/iiif/image/v3/handler/handler_test.go @@ -10,6 +10,7 @@ import ( "image/png" "io" "log/slog" + "net" "net/http" "net/http/httptest" "os" @@ -19,6 +20,7 @@ import ( "time" "github.com/libops/triplet/internal/cache" + "github.com/libops/triplet/internal/cors" "github.com/libops/triplet/internal/iiif/image/v3/pipeline" "github.com/libops/triplet/internal/iiif/image/v3/types" "github.com/libops/triplet/internal/storage" @@ -29,14 +31,14 @@ func setupTestServer(t *testing.T) (*httptest.Server, string) { } func setupTestServerWithCache(t *testing.T, derivCache cache.Store) (*httptest.Server, string) { - return setupTestServerWithOptions(t, derivCache, nil) + return setupTestServerWithOptions(t, derivCache, nil, "") } func setupTestServerWithAllowedOrigins(t *testing.T, allowedOrigins []string) (*httptest.Server, string) { - return setupTestServerWithOptions(t, cache.Noop{}, allowedOrigins) + return setupTestServerWithOptions(t, cache.Noop{}, allowedOrigins, "") } -func setupTestServerWithOptions(t *testing.T, derivCache cache.Store, allowedOrigins []string) (*httptest.Server, string) { +func setupTestServerWithOptions(t *testing.T, derivCache cache.Store, allowedOrigins []string, invalidationToken string) (*httptest.Server, string) { t.Helper() root := t.TempDir() img := image.NewRGBA(image.Rect(0, 0, 200, 100)) @@ -60,6 +62,9 @@ func setupTestServerWithOptions(t *testing.T, derivCache cache.Store, allowedOri pipeline.New(op, pipeline.Limits{MaxOutputPixels: 10_000_000}), derivCache, allowedOrigins, + invalidationToken, + nil, + nil, types.Limits{MaxArea: 10_000_000, MaxWidth: 4096, MaxHeight: 4096}, true, 250_000_000, @@ -223,6 +228,9 @@ func TestPipelineErrorUsesGenericResponse(t *testing.T) { pipeline.New(op, pipeline.Limits{MaxOutputPixels: 10_000_000}), cache.Noop{}, nil, + "", + nil, + nil, types.Limits{MaxArea: 10_000_000}, true, 250_000_000, @@ -275,6 +283,9 @@ func TestDerivativeCacheFailureWarns(t *testing.T) { pipeline.New(op, pipeline.Limits{MaxOutputPixels: 10_000_000}), failingCache{}, nil, + "", + nil, + nil, types.Limits{MaxArea: 10_000_000}, true, 250_000_000, @@ -501,6 +512,196 @@ func TestImageRequestETagChangesWhenSourceChanges(t *testing.T) { } } +func TestCacheInvalidationRequiresBearerToken(t *testing.T) { + store := newMemoryStore() + srv, _ := setupTestServerWithOptions(t, store, nil, "test-token") + defer srv.Close() + + resp, err := http.Post(srv.URL+"/iiif/3/sample.png/cache/invalidate", "application/json", nil) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d", resp.StatusCode) + } + if got := resp.Header.Get("WWW-Authenticate"); got != `Bearer realm="triplet-image-cache"` { + t.Fatalf("WWW-Authenticate = %q", got) + } +} + +func TestCacheInvalidationDisabledWithoutToken(t *testing.T) { + store := newMemoryStore() + srv, _ := setupTestServerWithOptions(t, store, nil, "") + defer srv.Close() + + resp, err := http.Post(srv.URL+"/iiif/3/sample.png/cache/invalidate", "application/json", nil) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d", resp.StatusCode) + } +} + +func TestImageRequestDoesNotCheckInvalidationMarkerWhenDisabled(t *testing.T) { + store := newMemoryStore() + store.data[invalidationMarkerKey("sample.png")] = []byte("unexpected-version") + store.meta[invalidationMarkerKey("sample.png")] = cache.Entry{StoredAt: time.Now()} + srv, _ := setupTestServerWithOptions(t, store, nil, "") + defer srv.Close() + + resp, err := http.Get(srv.URL + "/iiif/3/sample.png/full/max/0/default.png") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + firstETag := resp.Header.Get("ETag") + if firstETag == "" { + t.Fatal("missing first ETag") + } + + delete(store.data, invalidationMarkerKey("sample.png")) + delete(store.meta, invalidationMarkerKey("sample.png")) + resp, err = http.Get(srv.URL + "/iiif/3/sample.png/full/max/0/default.png") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("second status = %d", resp.StatusCode) + } + if got := resp.Header.Get("ETag"); got != firstETag { + t.Fatalf("ETag = %q, want %q", got, firstETag) + } +} + +func TestCacheInvalidationChangesDerivativeKey(t *testing.T) { + store := newMemoryStore() + srv, _ := setupTestServerWithOptions(t, store, nil, "test-token") + defer srv.Close() + + imageURL := srv.URL + "/iiif/3/sample.png/full/max/0/default.png" + first, err := http.Get(imageURL) + if err != nil { + t.Fatal(err) + } + defer first.Body.Close() + if first.StatusCode != http.StatusOK { + t.Fatalf("first status = %d", first.StatusCode) + } + firstETag := first.Header.Get("ETag") + if firstETag == "" { + t.Fatal("missing first ETag") + } + + second, err := http.Get(imageURL) + if err != nil { + t.Fatal(err) + } + defer second.Body.Close() + if second.StatusCode != http.StatusOK { + t.Fatalf("second status = %d", second.StatusCode) + } + if got := second.Header.Get("X-Cache"); got != "hit" { + t.Fatalf("second X-Cache = %q", got) + } + if got := second.Header.Get("ETag"); got != firstETag { + t.Fatalf("second ETag = %q, want %q", got, firstETag) + } + + req, err := http.NewRequest(http.MethodPost, srv.URL+"/iiif/3/sample.png/cache/invalidate", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer test-token") + invalidate, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer invalidate.Body.Close() + if invalidate.StatusCode != http.StatusNoContent { + t.Fatalf("invalidate status = %d", invalidate.StatusCode) + } + + third, err := http.Get(imageURL) + if err != nil { + t.Fatal(err) + } + defer third.Body.Close() + if third.StatusCode != http.StatusOK { + t.Fatalf("third status = %d", third.StatusCode) + } + if got := third.Header.Get("X-Cache"); got != "miss" { + t.Fatalf("third X-Cache = %q", got) + } + if got := third.Header.Get("ETag"); got == "" || got == firstETag { + t.Fatalf("third ETag = %q, first = %q", got, firstETag) + } +} + +func TestCacheInvalidationRejectsDisallowedCIDR(t *testing.T) { + store := newMemoryStore() + _, cidr, err := net.ParseCIDR("203.0.113.0/24") + if err != nil { + t.Fatal(err) + } + h := &Handler{ + prefix: "/iiif/3", + derivativeCache: store, + cors: cors.New(nil, exposeHeaders), + invalidationToken: "test-token", + invalidationCIDRs: []*net.IPNet{cidr}, + infoCache: map[string]cachedDimensions{}, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + req := httptest.NewRequest(http.MethodPost, "/iiif/3/sample.png/cache/invalidate", nil) + req.RemoteAddr = "198.51.100.10:12345" + req.Header.Set("Authorization", "Bearer test-token") + rec := httptest.NewRecorder() + + h.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d", rec.Code) + } +} + +func TestCacheInvalidationAllowsTrustedForwardedCIDR(t *testing.T) { + store := newMemoryStore() + _, allowed, err := net.ParseCIDR("203.0.113.0/24") + if err != nil { + t.Fatal(err) + } + _, trustedProxy, err := net.ParseCIDR("192.0.2.0/24") + if err != nil { + t.Fatal(err) + } + h := &Handler{ + prefix: "/iiif/3", + derivativeCache: store, + cors: cors.New(nil, exposeHeaders), + invalidationToken: "test-token", + invalidationCIDRs: []*net.IPNet{allowed}, + trustedProxies: []*net.IPNet{trustedProxy}, + infoCache: map[string]cachedDimensions{}, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + req := httptest.NewRequest(http.MethodPost, "/iiif/3/sample.png/cache/invalidate", nil) + req.RemoteAddr = "192.0.2.10:12345" + req.Header.Set("X-Forwarded-For", "203.0.113.9") + req.Header.Set("Authorization", "Bearer test-token") + rec := httptest.NewRecorder() + + h.ServeHTTP(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d", rec.Code) + } +} + type memoryStore struct { data map[string][]byte meta map[string]cache.Entry diff --git a/internal/server/server.go b/internal/server/server.go index c56265f..f393fdb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -47,6 +47,14 @@ func Build(cfg *config.Config, logger *slog.Logger) (*http.Server, error) { }() mux := http.NewServeMux() + trustedProxies, err := trustedProxyCIDRs(cfg.Logging.TrustedProxyCIDRs) + if err != nil { + return nil, err + } + imageInvalidationCIDRs, err := parseCIDRs("iiif.image.cache_invalidation_allowed_cidrs", cfg.IIIF.Image.CacheInvalidationAllowedCIDRs) + if err != nil { + return nil, err + } mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) @@ -93,6 +101,9 @@ func Build(cfg *config.Config, logger *slog.Logger) (*http.Server, error) { pipe, derivCache, imageAllowedOrigins(cfg), + cfg.IIIF.Image.CacheInvalidationToken, + imageInvalidationCIDRs, + trustedProxies, imgtypes.Limits{ MaxArea: cfg.IIIF.Image.MaxOutputPixels, MaxWidth: cfg.IIIF.Image.MaxWidth, @@ -142,10 +153,6 @@ func Build(cfg *config.Config, logger *slog.Logger) (*http.Server, error) { var handler http.Handler = mux handler = metrics.Middleware(handler) - trustedProxies, err := trustedProxyCIDRs(cfg.Logging.TrustedProxyCIDRs) - if err != nil { - return nil, err - } handler = observability.LoggingMiddleware(logger, observability.LoggingOptions{ TrustedProxies: trustedProxies, })(handler) @@ -202,11 +209,15 @@ func pprofHandler(token string, next http.HandlerFunc) http.HandlerFunc { } func trustedProxyCIDRs(raw []string) ([]*net.IPNet, error) { + return parseCIDRs("logging.trusted_proxy_cidrs", raw) +} + +func parseCIDRs(name string, raw []string) ([]*net.IPNet, error) { cidrs := make([]*net.IPNet, 0, len(raw)) for _, value := range raw { _, cidr, err := net.ParseCIDR(value) if err != nil { - return nil, fmt.Errorf("logging.trusted_proxy_cidrs: invalid CIDR %q: %w", value, err) + return nil, fmt.Errorf("%s: invalid CIDR %q: %w", name, value, err) } cidrs = append(cidrs, cidr) } diff --git a/internal/storage/local_url.go b/internal/storage/local_url.go index ebb3bba..740af80 100644 --- a/internal/storage/local_url.go +++ b/internal/storage/local_url.go @@ -56,9 +56,10 @@ type LocalURLMapping struct { } type authCacheEntry struct { - scope string - err error - expiresAt time.Time + scope string + identifier string + err error + expiresAt time.Time } var errAuthProbeHeadUnsupported = errors.New("auth probe head unsupported") @@ -351,7 +352,7 @@ func (l *LocalURLFallback) probeCached(ctx context.Context, key, identifier stri cacheable := cacheableAuthProbeResult(err) && cacheableAuthProbeResponse(err, respHeader, errorMinAge, time.Now()) l.debug(ctx, "local url auth probe completed", identifier, "cache_key", redact.Hash(key), "cacheable", cacheable, "ttl", ttl.String(), "err", err) if ttl > 0 && cacheable { - l.storeAuth(key, authCacheScope(key), err, ttl, maxEntries) + l.storeAuth(key, authCacheScope(key), identifier, err, ttl, maxEntries) } l.finishAuthProbe(key, err) return err @@ -452,14 +453,26 @@ func (l *LocalURLFallback) cachedAuth(key string) (error, bool) { return entry.err, true } -func (l *LocalURLFallback) storeAuth(key, scope string, err error, ttl time.Duration, maxEntries int) { +func (l *LocalURLFallback) storeAuth(key, scope, identifier string, err error, ttl time.Duration, maxEntries int) { l.authMu.Lock() defer l.authMu.Unlock() if l.auth == nil { l.auth = map[string]authCacheEntry{} } l.evictAuthLocked(scope, maxEntries) - l.auth[key] = authCacheEntry{scope: scope, err: err, expiresAt: time.Now().Add(ttl)} + l.auth[key] = authCacheEntry{scope: scope, identifier: identifier, err: err, expiresAt: time.Now().Add(ttl)} +} + +// InvalidateAuth clears cached authorization probe results for identifier. +func (l *LocalURLFallback) InvalidateAuth(_ context.Context, identifier string) error { + l.authMu.Lock() + defer l.authMu.Unlock() + for key, entry := range l.auth { + if entry.identifier == identifier { + delete(l.auth, key) + } + } + return nil } func (l *LocalURLFallback) evictAuthLocked(scope string, maxEntries int) { diff --git a/internal/storage/local_url_test.go b/internal/storage/local_url_test.go index c898827..13ca687 100644 --- a/internal/storage/local_url_test.go +++ b/internal/storage/local_url_test.go @@ -64,6 +64,28 @@ func TestLocalURLFallbackTriesLocalFileBeforeHTTP(t *testing.T) { } } +func TestLocalURLFallbackInvalidateAuthClearsIdentifierEntries(t *testing.T) { + op := &LocalURLFallback{ + auth: map[string]authCacheEntry{ + "one": {identifier: "sample.png", expiresAt: time.Now().Add(time.Hour)}, + "two": {identifier: "other.png", expiresAt: time.Now().Add(time.Hour)}, + "three": {identifier: "sample.png", expiresAt: time.Now().Add(time.Hour)}, + }, + } + if err := op.InvalidateAuth(context.Background(), "sample.png"); err != nil { + t.Fatal(err) + } + if _, ok := op.auth["one"]; ok { + t.Fatal("auth entry one was not invalidated") + } + if _, ok := op.auth["three"]; ok { + t.Fatal("auth entry three was not invalidated") + } + if _, ok := op.auth["two"]; !ok { + t.Fatal("unrelated auth entry was invalidated") + } +} + func TestLocalURLFallbackUsesHTTPWhenLocalMissing(t *testing.T) { root := t.TempDir() var httpHits int diff --git a/internal/storage/multiplex.go b/internal/storage/multiplex.go index c90bc54..d83103d 100644 --- a/internal/storage/multiplex.go +++ b/internal/storage/multiplex.go @@ -48,6 +48,20 @@ func (m *Multiplex) Meta(ctx context.Context, identifier string) (Meta, error) { return metaReader.Meta(ctx, identifier) } +// InvalidateAuth forwards auth-cache invalidation to the selected opener when +// it supports per-identifier auth state. +func (m *Multiplex) InvalidateAuth(ctx context.Context, identifier string) error { + opener, err := m.route(identifier) + if err != nil { + return err + } + invalidator, ok := opener.(AuthInvalidator) + if !ok { + return nil + } + return invalidator.InvalidateAuth(ctx, identifier) +} + func (m *Multiplex) route(identifier string) (Opener, error) { for _, r := range m.Routes { if r.HasPrefix != "" && strings.HasPrefix(identifier, r.HasPrefix) { diff --git a/internal/storage/opener.go b/internal/storage/opener.go index 5aab5e6..d7ed62f 100644 --- a/internal/storage/opener.go +++ b/internal/storage/opener.go @@ -68,3 +68,9 @@ type Opener interface { type MetaReader interface { Meta(ctx context.Context, identifier string) (Meta, error) } + +// AuthInvalidator clears cached authorization decisions for an identifier when +// a backend maintains per-source auth state. +type AuthInvalidator interface { + InvalidateAuth(ctx context.Context, identifier string) error +}