feat(discovery-catalog): add product image discovery pipeline#11
Merged
rovo79 merged 1 commit intoMay 12, 2026
Merged
Conversation
Add setup/scripts/discover-product-images.mjs — a Node.js script that discovers, fetches, optimises, and inventories product images from affiliate/source URLs. Key capabilities: - Reads a product-image-inventory CSV and attempts to discover primary product images from source URLs via og:image, meta tags, and structured data - Preserves existing optimised local assets: skips remote fetch when a local primary image already exists (reduces churn) - Optimises fetched images with sharp (resize, JPEG compression) and generates thumbnails - Tracks image provenance, checksums, and status per product - Detects retailer from URL domain for image attribution - Supports --limit and --key flags for incremental/targeted runs Requires: sharp (npm dependency) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Contributor
There was a problem hiding this comment.
Pull request overview
Adds a new setup-time CLI script to discover, download, optimize, and inventory product images for the discovery catalog workflow (build-time/static asset generation for the Astro frontend).
Changes:
- Introduces
setup/scripts/discover-product-images.mjsto process a product image inventory CSV and populate/public/catalog/products/<external_key>/with optimized images. - Implements image discovery from source pages (meta tags + JSON-LD + retailer-specific patterns), then downloads and optimizes images with
sharp. - Updates inventory rows with status/provenance/checksum/notes/last-checked metadata while preserving already-optimized local assets.
Comment on lines
+7
to
+8
| import sharp from 'sharp' | ||
|
|
Comment on lines
+543
to
+550
| try { | ||
| const { buffer, contentType, responseUrl } = await downloadImage(discoveredImageUrl) | ||
| const rawDir = path.join(context.cacheDir, 'raw', externalKey) | ||
| const optimizedDir = path.join(context.cacheDir, 'optimized', externalKey) | ||
| const publicDir = path.join(context.publicRoot, 'catalog', 'products', externalKey) | ||
| await ensureDir(rawDir) | ||
| await ensureDir(optimizedDir) | ||
| await ensureDir(publicDir) |
Comment on lines
+194
to
+228
| function inferRetailerName(url) { | ||
| if (!url) { | ||
| return '' | ||
| } | ||
| const host = cleanValue(new URL(url).hostname).replace(/^www\./, '') | ||
| const map = new Map([ | ||
| ['amazon.com', 'Amazon'], | ||
| ['amazon.co.uk', 'Amazon'], | ||
| ['bandai.co.jp', 'Bandai'], | ||
| ['banpresto.jp', 'Banpresto'], | ||
| ['bbts.com', 'BigBadToyStore'], | ||
| ['crunchyroll.com', 'Crunchyroll Store'], | ||
| ['etsy.com', 'Etsy'], | ||
| ['gamestop.com', 'GameStop'], | ||
| ]) | ||
|
|
||
| return map.get(host) ?? (host ? host.replace(/[-.]/g, ' ').replace(/\b\w/g, (match) => match.toUpperCase()) : '') | ||
| } | ||
|
|
||
| function inferProvenance(url) { | ||
| if (!url) { | ||
| return 'editorial_placeholder' | ||
| } | ||
| const host = cleanValue(new URL(url).hostname).replace(/^www\./, '') | ||
| if (!host || host.includes('example.com')) { | ||
| return 'editorial_placeholder' | ||
| } | ||
| if (host.includes('bandai') || host.includes('banpresto')) { | ||
| return 'manufacturer' | ||
| } | ||
| if (host.includes('official')) { | ||
| return 'official_press' | ||
| } | ||
| return 'retailer_listing' | ||
| } |
Comment on lines
+384
to
+406
| async function optimizeImage(buffer, destinationDir) { | ||
| const resize = async (size) => { | ||
| return sharp(buffer) | ||
| .rotate() | ||
| .flatten({ background: '#ffffff' }) | ||
| .resize({ | ||
| width: size, | ||
| height: size, | ||
| fit: 'contain', | ||
| background: '#ffffff', | ||
| withoutEnlargement: true, | ||
| }) | ||
| .jpeg({ quality: size >= 1000 ? 92 : 90, mozjpeg: true }) | ||
| .toBuffer() | ||
| } | ||
|
|
||
| const primary = await resize(1200) | ||
| const thumb = await resize(320) | ||
|
|
||
| await ensureDir(destinationDir) | ||
| await fs.writeFile(path.join(destinationDir, 'primary.jpg'), primary) | ||
| await fs.writeFile(path.join(destinationDir, 'thumb.jpg'), thumb) | ||
| } |
Comment on lines
+615
to
+634
| const inventory = await readCsv(inventoryPath) | ||
| const headers = mergeHeaders(REQUIRED_HEADERS, inventory.headers) | ||
| const filteredRows = inventory.rows.filter((row) => !options.key || cleanValue(row.external_key) === options.key) | ||
| const limit = Number.parseInt(options.limit, 10) | ||
| const selectedRows = Number.isFinite(limit) && limit > 0 ? filteredRows.slice(0, limit) : filteredRows | ||
|
|
||
| const processedRows = [] | ||
| for (const row of inventory.rows) { | ||
| const externalKey = cleanValue(row.external_key) | ||
| if (options.key && externalKey !== options.key) { | ||
| processedRows.push(row) | ||
| continue | ||
| } | ||
|
|
||
| if (selectedRows.includes(row)) { | ||
| processedRows.push(await processRow({ ...row }, { cacheDir, publicRoot })) | ||
| } else { | ||
| processedRows.push(row) | ||
| } | ||
| } |
Comment on lines
+364
to
+382
| async function downloadImage(url) { | ||
| const response = await fetch(url, { | ||
| headers: { | ||
| accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8', | ||
| 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0 Safari/537.36', | ||
| }, | ||
| redirect: 'follow', | ||
| }) | ||
|
|
||
| if (!response.ok) { | ||
| const error = new Error(`Image request failed: ${response.status} ${response.statusText}`) | ||
| error.status = response.status | ||
| throw error | ||
| } | ||
|
|
||
| const contentType = response.headers.get('content-type') ?? '' | ||
| const buffer = Buffer.from(await response.arrayBuffer()) | ||
| return { buffer, contentType, responseUrl: response.url || url } | ||
| } |
Comment on lines
+446
to
+448
| let discoveredImageUrl = manualOverrideUrl | ||
| let status = cleanValue(row.image_status) || 'missing' | ||
| let imageSourceUrl = cleanValue(row.image_source_url) |
Owner
Author
|
@copilot apply changes based on the comments in this thread |
Copilot stopped work on behalf of
rovo79 due to an error
May 12, 2026 11:15
rovo79
added a commit
that referenced
this pull request
May 13, 2026
feat(discovery-catalog): add product image discovery pipeline
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Summary
Adds a Node.js script for discovering, fetching, optimising, and inventorying product images from affiliate/source URLs. Depends on #9 (discovery catalog fields).
Changes
setup/scripts/discover-product-images.mjsA CLI tool that processes a product-image-inventory CSV and:
og:image) meta tagssharp(resize to max 1200px, JPEG 82% quality) and generates 400px thumbnailsCLI flags
--inventory <path>— path to the product image inventory CSV--cache-dir <path>— directory for raw/optimised image cache--public-root <path>— public assets directory for final placement--limit <n>— process only N products (for incremental runs)--key <id>— process a single product by external keyImage status lifecycle
missing→optimized_local(success) orblocked/needs_review(failure)Dependencies
Requires
sharpas an npm dependency.