From 80c638bdfd4f2cdfcbf91907407c15c161e5e373 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 23 Apr 2025 23:08:25 +0000 Subject: [PATCH 01/63] Refactor email field in CV files and enhance style setup for better input handling --- kadykov-cv-en.md | 2 +- kadykov-cv-fr.md | 4 +-- kadykov-letter-en.md | 2 +- kadykov-letter-fr.md | 2 +- style.typ | 79 +++++++++++++++++++++++++++----------------- 5 files changed, 53 insertions(+), 36 deletions(-) diff --git a/kadykov-cv-en.md b/kadykov-cv-en.md index 0e8c97f..53dcaff 100644 --- a/kadykov-cv-en.md +++ b/kadykov-cv-en.md @@ -1,7 +1,7 @@ --- author: Aleksandr KADYKOV title: Research Engineer -public-email: cv@kadykov.com +email: cv@kadykov.com github: kadykov gitlab: kadykov linkedin: aleksandr-kadykov diff --git a/kadykov-cv-fr.md b/kadykov-cv-fr.md index b046d91..eda9097 100644 --- a/kadykov-cv-fr.md +++ b/kadykov-cv-fr.md @@ -1,7 +1,7 @@ --- author: Aleksandr KADYKOV -title: Ingénieur de Recherche -public-email: cv@kadykov.com +title: Ingénieur de recherche +email: cv@kadykov.com github: kadykov gitlab: kadykov linkedin: aleksandr-kadykov diff --git a/kadykov-letter-en.md b/kadykov-letter-en.md index cf22e05..20daf6c 100644 --- a/kadykov-letter-en.md +++ b/kadykov-letter-en.md @@ -1,7 +1,7 @@ --- author: Aleksandr KADYKOV title: Letter to Future Employer -public-email: cv@kadykov.com +email: cv@kadykov.com github: kadykov gitlab: kadykov linkedin: aleksandr-kadykov diff --git a/kadykov-letter-fr.md b/kadykov-letter-fr.md index 621be18..e162f59 100644 --- a/kadykov-letter-fr.md +++ b/kadykov-letter-fr.md @@ -1,7 +1,7 @@ --- author: Aleksandr KADYKOV title: Lettre au Futur Employeur -public-email: cv@kadykov.com +email: cv@kadykov.com github: kadykov gitlab: kadykov linkedin: aleksandr-kadykov diff --git a/style.typ b/style.typ index 5809333..1f15467 100644 --- a/style.typ +++ b/style.typ @@ -11,27 +11,48 @@ #let full-width = 453.45pt #let bodywidth = 317.41pt #let line-spacing = 0.8em -// Define a function to set up style that accepts external variables +// Helper function to get value from sys.inputs or fallback +#let get-input(key, fallback) = { + if key in sys.inputs.keys() { + sys.inputs.at(key) + } else { + fallback + } +} + +// Define a function to set up style that accepts external variables (fallbacks) #let setup-style( - author: author, - public-email: "cv@kadykov.com", - title: "Research Engineer", - website: "www.kadykov.com", - github: "kadykov", - gitlab: "kadykov", - linkedin: "aleksandr-kadykov", - keywords: ("CV",), + // These values come from the Pandoc template ($variable$) and act as fallbacks + author: "Default Author", + email: "default@example.com", + title: "Default Title", + website: "example.com", + github: "default", + gitlab: "default", + linkedin: "default", + keywords: ("Default",), language: "en", date: auto, hyphenate: auto, doc, ) = { + // Prioritize sys.inputs (from --input args) over template variables (from YAML) + let effective-author = get-input("author", author) + let effective-email = get-input("email", email) + let effective-title = get-input("title", title) + let effective-website = get-input("website", website) + let effective-github = get-input("github", github) + let effective-gitlab = get-input("gitlab", gitlab) + let effective-linkedin = get-input("linkedin", linkedin) + // Keywords and date are less likely to be overridden via CLI, but could be added if needed + let effective-keywords = keywords + let effective-date = date // Document settings set text( font: "IBM Plex Serif", size: 10.5pt, - lang: language, + lang: language, // Language might need override too? get-input("language", language) fill: text-color, hyphenate: hyphenate, ) @@ -47,35 +68,30 @@ show strong: set text(font: "IBM Plex Serif SmBld") set document( - title: title, - author: author, - date: date, - keywords: keywords, + title: effective-title, + author: effective-author, + date: effective-date, + keywords: effective-keywords, ) set page( paper: "a4", margin: (x: 2.5cm, y: 3.0cm), header: [ - #author + #effective-author // Use effective value #h(1fr) - #if "PHONE" in sys.inputs.keys() [ - #link("tel:" + sys.inputs.PHONE.replace(regex("[^0-9+]"), ""))[ + // Phone only comes from sys.inputs (no YAML equivalent planned) + #if "phone" in sys.inputs.keys() [ + #link("tel:" + sys.inputs.phone.replace(regex("[^0-9+]"), ""))[ #text(fill: primary-color)[#fa-phone()] - #sys.inputs.PHONE.replace("_", " ") + #sys.inputs.phone.replace("_", " ") // Allow underscores for spacing ] | ] - #let email = { - if "EMAIL" in sys.inputs.keys() { - sys.inputs.EMAIL - } else { - public-email - } - } - #link("mailto:" + email)[ + // Use effective email (checks sys.inputs first, then YAML via template var) + #link("mailto:" + effective-email)[ #text(fill: primary-color)[#fa-envelope()] - #email + #effective-email ] #v(-0.5em) #line(length: 100%, stroke: 0.5pt) @@ -85,15 +101,16 @@ #v(-1em) #line(length: 100%, stroke: 0.5pt) #v(-0.5em) + // Use effective values for footer links #text(fill: primary-color)[#fa-arrow-up-right-from-square()] - #link("https://" + website)[#website] + #link("https://" + effective-website)[#effective-website] #h(1fr) #text(fill: black)[#fa-github()] - #link("https://github.com/" + github)[#github] | + #link("https://github.com/" + effective-github)[#effective-github] | #text(fill: orange)[#fa-gitlab()] - #link("https://gitlab.com/" + gitlab)[#gitlab] | + #link("https://gitlab.com/" + effective-gitlab)[#effective-gitlab] | #text(fill: blue)[#fa-linkedin()] - #link("https://www.linkedin.com/in/" + linkedin)[#linkedin] + #link("https://www.linkedin.com/in/" + effective-linkedin)[#effective-linkedin] ], ) From ff8a8df9c579d9d1a6ae2d03b5c551aa3dab2a51 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 23 Apr 2025 23:09:45 +0000 Subject: [PATCH 02/63] Add build script and update entrypoint in Dockerfile --- Dockerfile | 5 +- build.sh | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100755 build.sh diff --git a/Dockerfile b/Dockerfile index bc31160..89b340c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,9 @@ COPY --from=builder /fontawesome/*.typ /fontawesome/typst.toml ${TYPST_PACKAGE_P ENV TYPST_FONT_PATHS=/usr/share/fonts/ +COPY build.sh /usr/local/bin/build.sh +RUN chmod +x /usr/local/bin/build.sh + WORKDIR /data -ENTRYPOINT [ "just", "build" ] +ENTRYPOINT [ "build.sh" ] diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..5594f4e --- /dev/null +++ b/build.sh @@ -0,0 +1,153 @@ +#!/bin/sh +set -e + +# --- Default values --- +doc_type="" # cv or letter, inferred if empty +output_file="" +output_dir="." +input_file="" +typst_input_args="" +use_override_pipeline=false + +# --- Helper Functions --- +usage() { + echo "Usage: $0 [--type ] [--output ] [--output-dir ] [--set KEY=VALUE]..." + echo "" + echo " : Path to input Markdown file, or '-' for stdin." + echo " --type : Explicitly set document type (default: inferred from input filename, fallback 'cv')." + echo " --output : Output PDF file path, or '-' for stdout (default: based on input filename)." + echo " --output-dir : Directory for output file (default: '.'). Created if it doesn't exist." + echo " --set KEY=VALUE : Override metadata. KEY is converted to lowercase for Typst." + echo "" + echo "Overrides can also be provided via environment variables prefixed with TYPSTCV_" + echo "(e.g., TYPSTCV_EMAIL=me@private.com)." + exit 1 +} + +# --- Argument Parsing --- +while [ $# -gt 0 ]; do + case "$1" in + --type) + if [ -z "$2" ]; then echo "Error: --type requires an argument." >&2; usage; fi + if [ "$2" != "cv" ] && [ "$2" != "letter" ]; then echo "Error: --type must be 'cv' or 'letter'." >&2; usage; fi + doc_type="$2" + shift 2 + ;; + --output) + if [ -z "$2" ]; then echo "Error: --output requires an argument." >&2; usage; fi + output_file="$2" + shift 2 + ;; + --output-dir) + if [ -z "$2" ]; then echo "Error: --output-dir requires an argument." >&2; usage; fi + output_dir="$2" + shift 2 + ;; + --set) + if [ -z "$2" ]; then echo "Error: --set requires an argument (KEY=VALUE)." >&2; usage; fi + key=$(echo "$2" | cut -d= -f1 | tr '[:upper:]' '[:lower:]') + value=$(echo "$2" | cut -d= -f2-) + if [ -z "$key" ] || [ -z "$value" ]; then echo "Error: Invalid --set format. Use KEY=VALUE." >&2; usage; fi + typst_input_args="$typst_input_args --input $key=\"$value\"" + use_override_pipeline=true + shift 2 + ;; + --) # End of options + shift + break + ;; + -*) # Unknown option + echo "Error: Unknown option $1" >&2 + usage + ;; + *) # Input file + if [ -n "$input_file" ]; then echo "Error: Only one input file allowed." >&2; usage; fi + input_file="$1" + shift + ;; + esac +done + +# --- Input Validation --- +if [ -z "$input_file" ]; then + echo "Error: Input file or '-' (for stdin) is required." >&2 + usage +fi + +input_arg="$input_file" +if [ "$input_file" != "-" ] && [ ! -f "$input_file" ]; then + echo "Error: Input file not found: $input_file" >&2 + exit 1 +fi + +# --- Detect Environment Variable Overrides --- +for var in $(env | grep '^TYPSTCV_'); do + key=$(echo "$var" | cut -d= -f1 | sed 's/^TYPSTCV_//' | tr '[:upper:]' '[:lower:]') + value=$(echo "$var" | cut -d= -f2-) + if [ -n "$key" ] && [ -n "$value" ]; then + typst_input_args="$typst_input_args --input $key=\"$value\"" + use_override_pipeline=true + fi +done + +# --- Determine Document Type --- +if [ -z "$doc_type" ]; then + if echo "$input_file" | grep -q -i "cv"; then + doc_type="cv" + elif echo "$input_file" | grep -q -i "letter"; then + doc_type="letter" + else + echo "Warning: Could not infer document type from filename '$input_file'. Defaulting to 'cv'." >&2 + doc_type="cv" + fi +fi +template_file="typst-${doc_type}.typ" +echo "Info: Using document type: $doc_type (template: $template_file)" >&2 + +# --- Determine Output Path --- +output_arg="" +if [ "$output_file" = "-" ]; then + echo "Info: Outputting to stdout." >&2 + output_arg="-" + use_override_pipeline=true # Stdout requires the typst compile step +elif [ -n "$output_file" ]; then + # User specified exact output file + output_path="${output_dir}/${output_file}" + output_arg="-o $output_path" +else + # Default output filename based on input + if [ "$input_file" = "-" ]; then + echo "Error: Output file must be specified when reading from stdin (unless outputting to stdout with '--output -')." >&2 + usage + fi + base_name=$(basename "$input_file" .md) + output_path="${output_dir}/${base_name}.pdf" + output_arg="-o $output_path" +fi + +# Create output directory if needed and not outputting to stdout +if [ "$output_arg" != "-" ]; then + mkdir -p "$(dirname "$output_path")" + echo "Info: Output path: $output_path" >&2 +fi + + +# --- Build Commands --- +# Check if PANDOC_DATA_DIR is set, provide default if not (useful if run outside Docker) +PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-/usr/share/pandoc}" +pandoc_base="pandoc --data-dir=\"$PANDOC_DATA_DIR\" --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" + +# --- Execute Pipeline --- +echo "Info: Starting build..." >&2 +if [ "$use_override_pipeline" = true ]; then + echo "Info: Using override pipeline (Pandoc -> Typst)." >&2 + # Need eval to handle spaces in arguments correctly, especially $typst_input_args and paths + eval "$pandoc_base --to=typst --template=\"$template_file\" \"$input_arg\" | typst compile $typst_input_args - $output_arg" +else + echo "Info: Using direct pipeline (Pandoc -> PDF)." >&2 + # Need eval for paths + eval "$pandoc_base --template=\"$template_file\" \"$input_arg\" $output_arg" +fi + +echo "Info: Build finished successfully." >&2 +exit 0 From af4af57e7d378bcbe66c3a9de492856ecc9e3928 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 23 Apr 2025 23:10:20 +0000 Subject: [PATCH 03/63] Refactor project structure: replace `justfile` with `build.sh`, update Docker integration, and enhance README with new usage instructions --- README.md | 145 ++++++++++++++++++++++++++-------- memory-bank/activeContext.md | 24 ++++++ memory-bank/productContext.md | 21 +++++ memory-bank/progress.md | 37 +++++++++ memory-bank/projectbrief.md | 35 ++++++++ memory-bank/systemPatterns.md | 66 ++++++++++++++++ memory-bank/techContext.md | 39 +++++++++ 7 files changed, 332 insertions(+), 35 deletions(-) create mode 100644 memory-bank/activeContext.md create mode 100644 memory-bank/productContext.md create mode 100644 memory-bank/progress.md create mode 100644 memory-bank/projectbrief.md create mode 100644 memory-bank/systemPatterns.md create mode 100644 memory-bank/techContext.md diff --git a/README.md b/README.md index 3f6cff4..5225f38 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,79 @@ # Create CVs and Cover Letters in Markdown and Render to PDF with Typst and Pandoc -[![Build Status](https://github.com/kadykov/typstCV/actions/workflows/ci.yml/badge.svg)](https://github.com/kadykov/typstCV/actions/workflows/ci.yml) - ## Overview This project allows you to write your CV and cover letter in Markdown and generate PDFs using Typst and Pandoc. -It supports both **public** and **private** versions -(includes private details like phone numbers). +It provides a flexible command-line interface and Docker support for easy integration. --- -## Getting Started +## Usage -1. Clone the repository and open it as a development container. -2. Use the following commands for rendering: - - `just build`: Generates the **public version** (excludes private details like phone numbers). - - `just build-private`: Generates the **private version** (includes private details). +The primary way to use this tool is via the `build.sh` script or the provided Docker image. ---- +### Command-Line Interface (CLI) + +```bash +./build.sh [--type ] [--output ] [--output-dir ] [--set KEY=VALUE]... +``` + +**Arguments & Options:** + +- ``: (Required) Path to the input Markdown file, or `-` to read from stdin. +- `--type `: (Optional) Explicitly set the document type. If omitted, it attempts to infer from the input filename (containing "cv" or "letter"). Defaults to `cv` if inference fails. +- `--output `: (Optional) Specify the output PDF file path, or `-` to write to stdout. If omitted, defaults to `.pdf` in the output directory. *Required if reading from stdin and not outputting to stdout.* +- `--output-dir `: (Optional) Specify the directory for the output PDF file. Defaults to the current directory (`.`). The directory will be created if it doesn't exist. +- `--set KEY=VALUE`: (Optional, repeatable) Override metadata values. The `KEY` is converted to lowercase and passed to Typst. See [Metadata Overrides](#metadata-overrides). + +**Examples:** + +```bash +# Build a CV from a file, output to the same directory +./build.sh my-cv.md + +# Build a letter, specifying type and output directory +./build.sh my-letter.md --type letter --output-dir ./output + +# Build a CV, overriding email and adding phone number, output to specific file +./build.sh my-cv.md --set EMAIL=private@email.com --set PHONE="123 456 7890" --output my-private-cv.pdf + +# Build from stdin, output to stdout, overriding author +cat my-cv.md | ./build.sh - --set AUTHOR="Another Name" --output - > another-cv.pdf +``` + +### Docker Usage + +A Docker image containing all dependencies is available. + +1. **Pull the image from Docker Hub:** + ```bash + docker pull kadykov/typst-cv + ``` + *(Or build locally: `docker build -t kadykov/typst-cv .`)* + +2. **Run the container:** Mount your project directory (containing your `.md` files) to `/data` inside the container and pass arguments to `build.sh`. + + ```bash + docker run --rm -v "$(pwd):/data" kadykov/typst-cv [OPTIONS...] + ``` -## Generating Private Versions +**Docker Examples:** -By default, running `just build` creates a public version that omits sensitive details. -To create private versions: +```bash +# Build my-cv.md inside the container, output PDF appears in current host directory +docker run --rm -v "$(pwd):/data" kadykov/typst-cv my-cv.md -1. Rename `.env.example` to `.env`. -2. Enter your private details (phone number and email) in the `.env` file. -3. Run `just build-private` to render the private version. +# Build my-letter.md, output to ./output directory on host +docker run --rm -v "$(pwd):/data" kadykov/typst-cv my-letter.md --output-dir output + +# Build with overrides using environment variables (prefixed with TYPSTCV_) +docker run --rm -v "$(pwd):/data" \ + -e TYPSTCV_EMAIL="private@email.com" \ + -e TYPSTCV_PHONE="123 456 7890" \ + kadykov/typst-cv my-cv.md --output my-private-cv.pdf +``` --- @@ -41,18 +86,20 @@ Metadata and special features are defined in the YAML header. Include the following fields in the YAML header for your CV or cover letter: -**For both CV and cover letter:** +**Common Metadata Fields:** -- `author`: Your name. -- `title`: Job title or position. -- `public-email`: Publicly available email address. -- `github`, `gitlab`, `linkedin`: Your usernames on these platforms. -- `website`: Your website URL (without the protocol). -- `date`: Custom creation date (format: `year: YYYY, month: M, day: D`). -- `keywords`: Keywords for PDF metadata. -- `links`: Dictionary of keywords and URLs to auto-convert keywords into hyperlinks. +These fields are typically defined in the YAML frontmatter of your Markdown file. They can be overridden using the methods described in [Metadata Overrides](#metadata-overrides). -**Additional fields for cover letters:** +- `author`: Your name. +- `title`: Document title (e.g., "Research Engineer" for a CV, "Letter to..." for a cover letter). +- `email`: Your primary email address. +- `github`, `gitlab`, `linkedin`: Your usernames on these platforms. +- `website`: Your website URL (e.g., `www.kadykov.com`). +- `date`: Custom creation date (format: `year: YYYY, month: M, day: D`). Defaults to today if omitted. +- `keywords`: List of keywords for PDF metadata. +- `links`: Dictionary mapping keywords to URLs for automatic hyperlinking in the text (see [Features](#features)). + +**Additional Fields for Cover Letters:** - `to`: Recipient's address. - `from`: Your address. @@ -61,17 +108,45 @@ Include the following fields in the YAML header for your CV or cover letter: ## Features and Usage -### Adding Hyperlinks to Keywords +## Metadata Overrides + +You can override values defined in the YAML frontmatter without editing the Markdown file using two methods. This is useful for including private information (like a phone number) or customizing documents for specific applications. + +1. **Environment Variables:** Set environment variables prefixed with `TYPSTCV_`. The suffix will be used as the lowercase key for the override. + ```bash + export TYPSTCV_EMAIL="private@email.com" + export TYPSTCV_PHONE="+1 123 456 7890" + ./build.sh my-cv.md + # Or with Docker: + docker run --rm -v "$(pwd):/data" -e TYPSTCV_EMAIL="private@email.com" typst-cv-generator my-cv.md + ``` + +2. **`--set` Argument:** Use the `--set KEY=VALUE` argument when running `build.sh`. The `KEY` is converted to lowercase. + ```bash + ./build.sh my-cv.md --set email=private@email.com --set phone="+1 123 456 7890" --set title="Senior Engineer Application" + ``` + +**Priority:** Values from `--set` arguments and `TYPSTCV_` environment variables take precedence over values defined in the YAML frontmatter. The Typst template checks for these overrides first. + +**Note:** The `phone` field is typically *only* provided via overrides, as it's usually considered private information not included directly in the source Markdown/YAML. -A Pandoc filter (`linkify.lua`) converts keywords in your document into hyperlinks. -To enable: +--- + +## Features and Usage + +### Automatic Hyperlinks -1. Add a `links` dictionary in the YAML metadata. -2. Define keywords and their corresponding URLs. See [CV example](#cv-example) +A Pandoc filter (`linkify.lua`) automatically converts keywords found in your document text into hyperlinks if they are defined in the `links` dictionary in the YAML metadata. -The commands `just build` and `just build-private` apply this filter by default. +**Example YAML:** +```yaml +links: + TDS: https://en.wikipedia.org/wiki/Terahertz_time-domain_spectroscopy + THz: https://en.wikipedia.org/wiki/Terahertz_radiation +``` +Any occurrence of "TDS" or "THz" in the Markdown body will be linked accordingly. -### Right-Side Content in CVs +### Right-Side Content (CVs) You can display content (e.g., profile picture, location, dates) @@ -139,13 +214,13 @@ and they automatically span the full width of the document. Below is a minimal example of a CV in Markdown. For a detailed example, see [`kadykov-cv-en.md`](kadykov-cv-en.md). -Rendered PDFs are [hosted](http://github.kadykov.com/typstCV/) on GitHub Pages. +Rendered example PDFs are available [here](http://github.kadykov.com/typstCV/). ```markdown --- author: Aleksandr KADYKOV title: Research Engineer -public-email: cv@kadykov.com +email: cv@kadykov.com github: kadykov gitlab: kadykov linkedin: aleksandr-kadykov diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md new file mode 100644 index 0000000..e74b1ac --- /dev/null +++ b/memory-bank/activeContext.md @@ -0,0 +1,24 @@ +# Active Context: Interface Refactoring Complete (2025-04-23) + +## Current Focus + +The refactoring of the project's user interface is complete. The `justfile` has been replaced with `build.sh`, providing a flexible command-line interface with support for overrides and stdin/stdout. Docker integration has been updated accordingly. + +## Recent Actions (Completed) + +- Reviewed project structure, `README.md`, `justfile`, Lua filters (`linkify.lua`, `typst-cv.lua`), Typst templates (`typst-cv.typ`, `typst-letter.typ`), style package (`style.typ`, `typst.toml`), `Dockerfile`, and `entrypoint.sh`. +- Confirmed the rendering pipeline (Markdown -> Pandoc w/ Filters & Template -> Typst -> PDF). +- Identified the `justfile`'s hardcoded values and rigid structure as the main usability bottleneck. +- Analyzed the mechanism for handling private data (`.env` + `just build-private` piping to `typst compile --input`) and identified it as inflexible. +- Examined Docker setup and identified the need to update the `ENTRYPOINT`. + +1. **Modified `style.typ`:** Implemented universal override logic (checking `sys.inputs` first) and renamed `public-email` to `email`. +2. **Created `build.sh`:** Implemented the new flexible command-line interface with argument parsing, stdin/stdout support, override detection (`TYPSTCV_*`, `--set`), and dynamic pipeline selection. +3. **Updated `Dockerfile`:** Changed `ENTRYPOINT` to `build.sh` and added steps to copy/chmod the script. +4. **Updated Documentation:** Modified `README.md` with new usage instructions for `build.sh` (CLI & Docker, including correct Docker Hub tag `kadykov/typst-cv`) and updated example files (`kadykov-*.md`) to use `email`. +5. **Cleaned Up:** Removed `justfile`, `action.yml`, and `entrypoint.sh`. +6. **Updated Memory Bank:** Updated `activeContext.md` and `progress.md`. + +## Immediate Next Steps + +- Present the completed work to the user. diff --git a/memory-bank/productContext.md b/memory-bank/productContext.md new file mode 100644 index 0000000..782da18 --- /dev/null +++ b/memory-bank/productContext.md @@ -0,0 +1,21 @@ +# Product Context: Typst CV/Letter Generator + +## Problem Solved + +The project aims to simplify the creation of professional CVs and cover letters. Users can focus on content by writing in familiar Markdown, while leveraging the powerful typesetting capabilities of Typst for high-quality PDF output. It avoids the need for users to learn complex Typst syntax directly for basic document creation. + +## Core User Need + +Users need a straightforward way to convert their Markdown CV/letter documents into PDFs without dealing with complex build commands or rigid file naming conventions. The original `justfile`-based system, while functional, was identified as too complex and inflexible for general use. + +## Desired User Experience + +- **Simplicity:** A single, intuitive command should handle the conversion process. +- **Flexibility:** Users should be able to name their Markdown files freely. +- **Customization:** Easy mechanism to provide private information (like phone/email) or override other metadata without editing core project files. +- **Standard Tooling:** Ability to integrate the tool into scripts (stdin/stdout) and use it easily within a Docker container. +- **Clear Documentation:** Instructions should be clear for both direct command-line usage and Docker usage. + +## Target Audience + +Individuals (developers, researchers, etc.) who are comfortable with Markdown and prefer text-based document creation workflows, but want polished PDF outputs without mastering a full typesetting system like LaTeX or Typst directly. diff --git a/memory-bank/progress.md b/memory-bank/progress.md new file mode 100644 index 0000000..8eaa6e4 --- /dev/null +++ b/memory-bank/progress.md @@ -0,0 +1,37 @@ +# Progress: Typst CV/Letter Generator (As of 2025-04-23) + +## What Works (Current State) + +- The core Markdown-to-PDF pipeline using Pandoc, Lua filters, and Typst functions correctly for the example files (`kadykov-*.md`). +- Rendering of both CVs and Letters works. +- Custom Markdown features (YAML metadata, `links`, attributes like `{photo}`, `{date}`, `{location}`, class `.hidden`) are implemented and functional via Lua filters and Typst styles. +- Public vs. Private rendering using `.env` and `just build`/`just build-private` works, albeit inflexibly. +- Docker image builds successfully and contains all necessary dependencies and project files. +- GitHub Action (`entrypoint.sh`) exists but relies on the current `justfile`. + +## What's Left to Build (Current Task) + +1. **`build.sh` Script:** Create the new command-line interface script. + - Argument parsing (input, output, type, overrides). + - Stdin/stdout support. + - Override detection (`TYPSTCV_*` env vars, `--set` args). + - Dynamic pipeline selection (Pandoc-direct-PDF vs. Pandoc-Typst | Typst-Compile). + * Command construction and execution. + * Error handling and user feedback. +2. **`Dockerfile` Update:** Modify the `ENTRYPOINT` to use `build.sh`. Copy `build.sh` into the image. +3. **`entrypoint.sh` Update:** Modify the GitHub Action script to use `build.sh` instead of `just build`. +4. **`README.md` Update:** Rewrite usage instructions for `build.sh` (CLI and Docker). +5. **`justfile` Removal:** Delete the old `justfile`. + +## Current Status + +- **Planning Phase Complete:** Analysis of the existing system is done. A detailed plan for refactoring the interface with `build.sh` has been agreed upon. +- **Memory Bank Established:** Initial versions of all core Memory Bank files have been created. +- **Ready for Implementation:** The next step is to draft the `build.sh` script. + +## Known Issues (Existing System) + +- **Rigid Interface:** `justfile` requires specific filenames (`kadykov-*`) and structure. +- **Inflexible Private Data:** `.env` + `build-private` is the only way to include private info. +- **Docker Entrypoint:** Tied to the inflexible `justfile`. +- **Minor Layout Quirk:** README mentions side content (`{date}`, `{location}`) starting a new paragraph after the heading, potentially causing extra space. (This is a Typst/styling detail, not directly addressed by the current refactoring task, but noted). diff --git a/memory-bank/projectbrief.md b/memory-bank/projectbrief.md new file mode 100644 index 0000000..07d135f --- /dev/null +++ b/memory-bank/projectbrief.md @@ -0,0 +1,35 @@ +# Project Brief: Typst CV/Letter Generator + +## Core Goal + +To provide a flexible and user-friendly system for rendering Curriculum Vitae (CV) and Cover Letters written in Markdown format into professional-looking PDFs using Typst. + +## Key Features (Original) + +- **Input Format:** Markdown with YAML frontmatter for metadata. +- **Custom Attributes:** Uses Markdown attributes for specific layout features (e.g., `photo`, `date`, `location`). +- **Rendering Pipeline:** + 1. Markdown + YAML -> Pandoc -> Typst format. + 2. Pandoc uses custom Lua filter (`linkify.lua`) for hyperlink enrichment. + 3. Pandoc uses custom Lua filter (`typst-cv.lua`) to translate attributes/classes to Typst functions. + 4. Pandoc uses specific templates (`typst-cv.typ`, `typst-letter.typ`). + 5. Typst renders the generated `.typ` file to PDF, using common styles from `style.typ` (defined as local package `@local/pandoc-cv`). +- **Workflow Management:** `justfile` orchestrates the multi-step rendering process (identified as problematic). +- **Distribution:** `Dockerfile` packages all dependencies (Pandoc, Typst, Just, fonts) for setup and execution. Includes a GitHub Action entrypoint (`entrypoint.sh`). + +## Problem Statement (User Identified & Analyzed) + +- The primary interface via `justfile` commands is complex, rigid, and not user-friendly. +- Hardcoded filenames (`kadykov`) and reliance on strict naming conventions (`filename-type-lang.md`) in `justfile` prevent general use. +- Repetitive recipes in `justfile` for public/private and languages. +- The distinction between public/private builds via `.env` and `--private` flag is inflexible. +- Docker entrypoint (`Dockerfile`) is tied to the inflexible `justfile`. + +## Objective of Current Task (Refined) + +1. **Refactor Interface:** Replace the `justfile` with a flexible shell script (`build.sh`). +2. **Improve Usability:** Design `build.sh` with a clear command-line interface accepting input files, output paths, and options. +3. **Flexible Overrides:** Implement a system for overriding YAML metadata using environment variables (prefixed `TYPSTCV_*`) and command-line arguments (`--set KEY=VALUE`), replacing the rigid `--private` flag. +4. **Add Features:** Support reading from stdin and writing PDF to stdout. +5. **Docker Integration:** Update `Dockerfile` entrypoint to use `build.sh`, making the container directly usable with the new interface. Update the GitHub Action script (`entrypoint.sh`) to use `build.sh`. +6. **Documentation:** Update `README.md` with new usage instructions. Establish and maintain the Memory Bank. diff --git a/memory-bank/systemPatterns.md b/memory-bank/systemPatterns.md new file mode 100644 index 0000000..c466590 --- /dev/null +++ b/memory-bank/systemPatterns.md @@ -0,0 +1,66 @@ +# System Patterns: Typst CV/Letter Generator + +## Core Architecture: Pipeline Conversion + +The system employs a multi-stage pipeline pattern to convert Markdown input into a final PDF output. + +```mermaid +graph LR + A[Markdown + YAML Frontmatter] --> B(Pandoc); + C[linkify.lua] --> B; + D[typst-cv.lua] --> B; + E[typst-cv.typ / typst-letter.typ Template] --> B; + B -- Typst Code --> F(Typst Compile); + G[style.typ / @local/pandoc-cv] --> F; + H[Environment Variables / --set Args] -.-> F; + F --> I[PDF Output]; + + subgraph Pandoc Conversion + B + C + D + E + end + + subgraph Typst Rendering + F + G + H + end +``` + +## Key Components & Roles + +1. **Markdown Input:** User provides content and basic metadata (YAML). Uses specific Markdown attributes (`{key=value}`) and classes (`{.class}`) for layout hints. +2. **Pandoc:** Acts as the central converter. + * Parses Markdown and YAML. + * Applies Lua filters for transformations. + * Uses a Typst template (`.typ`) to structure the output. + * Can output either final PDF (simpler cases) or intermediate Typst code. +3. **Lua Filters (`linkify.lua`, `typst-cv.lua`):** Extend Pandoc's functionality. + * `linkify.lua`: Automatically creates hyperlinks based on `links` metadata. + * `typst-cv.lua`: Translates Markdown attributes/classes into specific Typst function calls (e.g., `#body-side`, `#hidden-heading`), decoupling Markdown syntax from Typst implementation details. +4. **Typst Templates (`typst-cv.typ`, `typst-letter.typ`):** Basic structure files used by Pandoc. They primarily import the main style package and pass metadata. +5. **Typst Style Package (`style.typ` / `@local/pandoc-cv`):** Contains the core Typst logic. + * Defines document structure, page layout, fonts, colors. + * Implements the functions called by `typst-cv.lua` (e.g., `body-side`, `hidden-heading`). + * Handles metadata passed from templates (`setup-style` function). + * Accesses override variables (`sys.inputs`) provided during Typst compilation. +6. **Build Script (`build.sh` - Proposed):** Orchestrates the pipeline. + * Parses user arguments (input file, overrides, output options). + * Selects the appropriate Pandoc template. + * Detects override variables (`TYPSTCV_*` env vars, `--set` args). + * Chooses the correct pipeline (direct Pandoc-to-PDF or Pandoc-to-Typst | Typst Compile) based on whether overrides are needed or stdout is used. + * Constructs and executes the final `pandoc` and `typst` commands. +7. **Docker (`Dockerfile`):** Packages the entire environment (dependencies, filters, templates, styles, build script) for reproducible execution. The entrypoint will be set to `build.sh`. + +## Key Design Decisions (Current & Proposed) + +- **Markdown as Source:** Prioritizes ease of writing for the user. +- **Pandoc as Bridge:** Leverages Pandoc's robust Markdown parsing and filter capabilities. +- **Lua Filters for Customization:** Encapsulates custom Markdown-to-Typst translation logic. +- **Typst for Layout:** Utilizes Typst's modern typesetting features for high-quality output. +- **Decoupled Styling:** Centralizes Typst styling logic in `style.typ` (via local package). +- **Script-based Interface (Proposed):** Replaces rigid `justfile` with a flexible `build.sh` for better usability and integration. +- **Environment/Arg Overrides (Proposed):** Provides a standard mechanism (`TYPSTCV_*`, `--set`) for customizing output without modifying source files. +- **Docker for Distribution:** Ensures consistent environment and simplifies setup. diff --git a/memory-bank/techContext.md b/memory-bank/techContext.md new file mode 100644 index 0000000..17c7986 --- /dev/null +++ b/memory-bank/techContext.md @@ -0,0 +1,39 @@ +# Technical Context: Typst CV/Letter Generator + +## Core Technologies + +- **Pandoc:** Universal document converter. Used for Markdown parsing, YAML metadata extraction, Lua filter execution, and applying Typst templates. Version specified in `Dockerfile`. +- **Typst:** Modern typesetting system. Used for final PDF rendering. Version specified in `Dockerfile`. +- **Lua:** Scripting language used for Pandoc filters (`linkify.lua`, `typst-cv.lua`). Filters are executed by Pandoc. +- **Markdown:** Input format (CommonMark with Pandoc extensions like YAML frontmatter, attributes, classes). +- **YAML:** Used within Markdown frontmatter for metadata (`author`, `title`, `links`, etc.). +- **Shell Script (Bash/sh):** Proposed for `build.sh` to replace `justfile`. Needs to be POSIX-compliant for broader compatibility (especially within minimal Docker containers like Alpine, though the current base is Fedora). +- **Docker:** Containerization platform used for packaging and distribution. `Dockerfile` defines the image build process. +- **Just:** Task runner (currently used, to be replaced by `build.sh`). Version specified in `Dockerfile`. + +## Key Files & Locations (within Docker image) + +- **Pandoc Filters (`*.lua`):** Copied to `$PANDOC_DATA_DIR/filters/` (e.g., `/usr/share/pandoc-3.1.11.1/filters/`). +- **Pandoc Templates (`typst-*.typ`):** Copied to `$PANDOC_DATA_DIR/data/templates/` (e.g., `/usr/share/pandoc-3.1.11.1/data/templates/`). +- **Typst Style Package (`style.typ`, `typst.toml`):** Copied to `$TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0/` (e.g., `/usr/local/share/typst/packages/local/pandoc-cv/0.1.0/`). +- **Typst Executable:** `/usr/bin/typst`. +- **Fonts:** Installed via `dnf` into system font directories (e.g., `/usr/share/fonts/`). `TYPST_FONT_PATHS` environment variable points Typst to these. +- **Build Script (`build.sh` - Proposed):** To be copied likely to `/usr/local/bin/`. +- **Working Directory:** `/data` (users mount their project here). + +## Dependencies & Versions + +- Specific versions of Fedora, Alpine (for builder stage), Pandoc, Just, Typst, and fonts are pinned in the `Dockerfile`. This ensures reproducibility. +- External Typst packages (`@preview/fontawesome`) are also versioned. + +## Build/Execution Environment + +- Primarily designed to run within the provided Docker container, which includes all necessary dependencies and configurations. +- The proposed `build.sh` script relies on standard POSIX shell commands and the installed tools (Pandoc, Typst). +- Environment variables (`TYPSTCV_*`, potentially `PANDOC_DATA_DIR`, `TYPST_PACKAGE_PATH`, `TYPST_FONT_PATHS`) play a role in configuration. + +## Constraints + +- The system relies on specific versions of tools being available. +- Correct placement of filters, templates, and packages within the Docker image (or host system if run outside Docker) is crucial. +- Font availability is essential for correct rendering. From fe2154004e57c3d8fc39ae18de5261bb0dbe226e Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 23 Apr 2025 23:11:21 +0000 Subject: [PATCH 04/63] Remove deprecated files: delete action.yml, entrypoint.sh, and justfile --- action.yml | 14 ---------- entrypoint.sh | 19 -------------- justfile | 72 --------------------------------------------------- 3 files changed, 105 deletions(-) delete mode 100644 action.yml delete mode 100755 entrypoint.sh delete mode 100644 justfile diff --git a/action.yml b/action.yml deleted file mode 100644 index 8a3d789..0000000 --- a/action.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -name: 'Typst CV' -description: 'Render PDF files with Typst' -inputs: - output-folder: - description: 'Output folder' - required: false - default: 'public' -runs: - using: 'docker' - image: 'Dockerfile' - entrypoint: './entrypoint.sh' - env: - OUTPUT_FOLDER: ${{ inputs.output-folder }} diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index 43b6457..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -l - -set -e - -# Render default PDF files -just build - -# Use the correct environment variable for the input -output_folder="${OUTPUT_FOLDER}" - -# Ensure the output folder exists -mkdir -p "$output_folder" - -# Copy generated files to the output folder -cp *.pdf *.html "$output_folder/" - -# List files in the output folder and print them -files=$(ls "$output_folder/") -echo -e "Files in the $output_folder folder:\n$files" diff --git a/justfile b/justfile deleted file mode 100644 index 532662d..0000000 --- a/justfile +++ /dev/null @@ -1,72 +0,0 @@ -set dotenv-load -filename := "kadykov" -cv := "cv" -letter := "letter" -english := "en" -french := "fr" -output-dir := "." -typst := "typst compile" -pandoc := "pandoc --data-dir=$PANDOC_DATA_DIR --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" -pandoc-to-typst := "--to=typst | typst compile -" -private-args := '--input EMAIL="$EMAIL" --input PHONE="$PHONE"' - - -build: - just english - just french - -build-private: - just english-private - just french-private - -english: - mkdir -p {{output-dir}} - {{pandoc}} \ - {{filename}}-{{cv}}-{{english}}.md \ - -o {{output-dir}}/{{filename}}-{{cv}}-{{english}}.pdf \ - --template=typst-{{cv}}.typ - {{pandoc}} \ - {{filename}}-{{letter}}-{{english}}.md \ - -o {{output-dir}}/{{filename}}-{{letter}}-{{english}}.pdf \ - --template=typst-{{letter}}.typ - -english-private: - mkdir -p {{output-dir}} - {{pandoc}} \ - {{filename}}-{{cv}}-{{english}}.md \ - --template=typst-{{cv}}.typ \ - {{pandoc-to-typst}} \ - {{output-dir}}/{{filename}}-{{cv}}-{{english}}.pdf \ - {{private-args}} - {{pandoc}} \ - {{filename}}-{{letter}}-{{english}}.md \ - --template=typst-{{letter}}.typ \ - {{pandoc-to-typst}} \ - {{output-dir}}/{{filename}}-{{letter}}-{{english}}.pdf \ - {{private-args}} - -french: - mkdir -p {{output-dir}} - {{pandoc}} \ - {{filename}}-{{cv}}-{{french}}.md \ - -o {{output-dir}}/{{filename}}-{{cv}}-{{french}}.pdf \ - --template=typst-{{cv}}.typ - {{pandoc}} \ - {{filename}}-{{letter}}-{{french}}.md \ - -o {{output-dir}}/{{filename}}-{{letter}}-{{french}}.pdf \ - --template=typst-{{letter}}.typ - -french-private: - mkdir -p {{output-dir}} - {{pandoc}} \ - {{filename}}-{{cv}}-{{french}}.md \ - --template=typst-{{cv}}.typ \ - {{pandoc-to-typst}} \ - {{output-dir}}/{{filename}}-{{cv}}-{{french}}.pdf \ - {{private-args}} - {{pandoc}} \ - {{filename}}-{{letter}}-{{french}}.md \ - --template=typst-{{letter}}.typ \ - {{pandoc-to-typst}} \ - {{output-dir}}/{{filename}}-{{letter}}-{{french}}.pdf \ - {{private-args}} From d19b861deddebc88ef89acba7b89f92f0650492e Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 23 Apr 2025 23:11:49 +0000 Subject: [PATCH 05/63] Enhance devcontainer configuration: add volume mount for vscode home and include additional VS Code extension --- .devcontainer/devcontainer.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 07c2e71..2a697a5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -26,8 +26,16 @@ "GitHub.vscode-github-actions", "eamodio.gitlens", "sumneko.lua", - "yinfei.luahelper" + "yinfei.luahelper", + "saoudrizwan.claude-dev" ] } - } + }, + "mounts": [ + { + "source": "vscode-home", + "target": "/home/vscode", + "type": "volume" + } + ] } From 95941056868928ede769b1945ce89e43eb5887e5 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 24 Apr 2025 10:24:06 +0000 Subject: [PATCH 06/63] Refactor email handling: replace `public-email` with `email` in CV and letter templates --- typst-cv.typ | 2 +- typst-letter.typ | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/typst-cv.typ b/typst-cv.typ index d101284..e989004 100644 --- a/typst-cv.typ +++ b/typst-cv.typ @@ -7,7 +7,7 @@ #show: setup-style.with( $if(title)$title: "$title$",$endif$ $if(author)$author: "$author$",$endif$ - $if(public-email)$public-email: "$public-email$".replace("\\", ""),$endif$ + $if(email)$email: "$email$".replace("\\", ""),$endif$ $if(github)$github: "$github$",$endif$ $if(gitlab)$gitlab: "$gitlab$",$endif$ $if(linkedin)$linkedin: "$linkedin$",$endif$ diff --git a/typst-letter.typ b/typst-letter.typ index a2bf92d..08b8682 100644 --- a/typst-letter.typ +++ b/typst-letter.typ @@ -19,7 +19,7 @@ #show: setup-style.with( $if(title)$title: "$title$",$endif$ $if(author)$author: "$author$",$endif$ - $if(public-email)$public-email: "$public-email$".replace("\\", ""),$endif$ + $if(email)$email: "$email$".replace("\\", ""),$endif$ $if(github)$github: "$github$",$endif$ $if(gitlab)$gitlab: "$gitlab$",$endif$ $if(linkedin)$linkedin: "$linkedin$",$endif$ From 8dd536690050ea5cd8a9fe2ad804eb15a09a61a3 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:23:56 +0000 Subject: [PATCH 07/63] Refactor date handling: replace direct date usage with constructed date-object in CV and letter templates --- typst-cv.typ | 28 ++++++++++++---------------- typst-letter.typ | 34 +++++++++++++++------------------- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/typst-cv.typ b/typst-cv.typ index e989004..dac50a0 100644 --- a/typst-cv.typ +++ b/typst-cv.typ @@ -1,25 +1,21 @@ // #import "style.typ": * #import "@local/pandoc-cv:0.1.0": * -#let date = $if(date)$datetime($date$)$else$datetime.today()$endif$ +// Construct datetime object from structured date if provided via YAML map, else use today +#let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ // Call the function from `style.typ` and pass variables to set up the document style #show: setup-style.with( - $if(title)$title: "$title$",$endif$ - $if(author)$author: "$author$",$endif$ - $if(email)$email: "$email$".replace("\\", ""),$endif$ - $if(github)$github: "$github$",$endif$ - $if(gitlab)$gitlab: "$gitlab$",$endif$ - $if(linkedin)$linkedin: "$linkedin$",$endif$ - $if(website)$website: "$website$",$endif$ - $if(date)$date: date,$endif$ - $if(keywords)$keywords: ( - $for(keywords)$ - "$keywords$", - $endfor$ - ), - $endif$ - hyphenate: auto, + $if(title)$title: "$title$", $endif$ // Comma after if present + $if(author)$author: "$author$", $endif$ // Comma after if present + $if(email)$email: "$email$".replace("\\", ""), $endif$ // Comma after if present + $if(github)$github: "$github$", $endif$ // Comma after if present + $if(gitlab)$gitlab: "$gitlab$", $endif$ // Comma after if present + $if(linkedin)$linkedin: "$linkedin$", $endif$ // Comma after if present + $if(website)$website: "$website$", $endif$ // Comma after if present + date: date-object, // Always present, comma needed before next potential arg + $if(keywords)$keywords: ($for(keywords)$ "$keywords$", $endfor$), $endif$ // Comma after if present + hyphenate: auto // Last argument, no trailing comma ) #block(width: bodywidth)[ diff --git a/typst-letter.typ b/typst-letter.typ index 08b8682..eb397d1 100644 --- a/typst-letter.typ +++ b/typst-letter.typ @@ -1,9 +1,10 @@ #import "@local/pandoc-cv:0.1.0": * -#let date = $if(date)$datetime($date$)$else$datetime.today()$endif$ +// Construct datetime object from structured date if provided via YAML map, else use today +#let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ #let suffix = { - let day = date.day() + let day = date-object.day() // Use date-object if day == 1 or day == 21 or day == 31 { "st" } else if day == 2 or day == 22 { @@ -17,30 +18,25 @@ // Call the function from `style.typ` and pass variables to set up the document style #show: setup-style.with( - $if(title)$title: "$title$",$endif$ - $if(author)$author: "$author$",$endif$ - $if(email)$email: "$email$".replace("\\", ""),$endif$ - $if(github)$github: "$github$",$endif$ - $if(gitlab)$gitlab: "$gitlab$",$endif$ - $if(linkedin)$linkedin: "$linkedin$",$endif$ - $if(website)$website: "$website$",$endif$ - $if(date)$date: date,$endif$ - $if(keywords)$keywords: ( - $for(keywords)$ - "$keywords$", - $endfor$ - ), - $endif$ - hyphenate: true, + $if(title)$title: "$title$", $endif$ // Comma after if present + $if(author)$author: "$author$", $endif$ // Comma after if present + $if(email)$email: "$email$".replace("\\", ""), $endif$ // Comma after if present + $if(github)$github: "$github$", $endif$ // Comma after if present + $if(gitlab)$gitlab: "$gitlab$", $endif$ // Comma after if present + $if(linkedin)$linkedin: "$linkedin$", $endif$ // Comma after if present + $if(website)$website: "$website$", $endif$ // Comma after if present + date: date-object, // Use the constructed date-object, comma needed before next potential arg + $if(keywords)$keywords: ($for(keywords)$ "$keywords$", $endfor$), $endif$ // Comma after if present + hyphenate: true // Last argument, no trailing comma ) #let from-content = [ $if(from)$$from$$endif$ - #date.display( + #date-object.display( // Use date-object "[month repr:long] [day padding:none]" )#super(suffix), - #date.year() + #date-object.year() // Use date-object ] #let to-content = [$if(to)$$to$$endif$] From 742596cccd7a4df93cdd7fe7bf064f5330fca96f Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:25:41 +0000 Subject: [PATCH 08/63] Revert output format changes: restore output_format handling and update usage instructions in build script --- build.sh | 49 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/build.sh b/build.sh index 5594f4e..f27a149 100755 --- a/build.sh +++ b/build.sh @@ -6,17 +6,20 @@ doc_type="" # cv or letter, inferred if empty output_file="" output_dir="." input_file="" +# output_format="pdf" # Reverted: Removed output_format typst_input_args="" use_override_pipeline=false # --- Helper Functions --- usage() { + # Reverted: Removed --output-format from usage echo "Usage: $0 [--type ] [--output ] [--output-dir ] [--set KEY=VALUE]..." echo "" echo " : Path to input Markdown file, or '-' for stdin." echo " --type : Explicitly set document type (default: inferred from input filename, fallback 'cv')." - echo " --output : Output PDF file path, or '-' for stdout (default: based on input filename)." + echo " --output : Output PDF file path, or '-' for stdout (default: based on input filename)." echo " --output-dir : Directory for output file (default: '.'). Created if it doesn't exist." + # echo " --output-format : Format for the output file (default: 'pdf')." echo " --set KEY=VALUE : Override metadata. KEY is converted to lowercase for Typst." echo "" echo "Overrides can also be provided via environment variables prefixed with TYPSTCV_" @@ -43,6 +46,13 @@ while [ $# -gt 0 ]; do output_dir="$2" shift 2 ;; + # Reverted: Removed --output-format parsing + # --output-format) + # if [ -z "$2" ]; then echo "Error: --output-format requires an argument." >&2; usage; fi + # if [ "$2" != "pdf" ] && [ "$2" != "typst" ]; then echo "Error: --output-format must be 'pdf' or 'typst'." >&2; usage; fi + # output_format="$2" + # shift 2 + # ;; --set) if [ -z "$2" ]; then echo "Error: --set requires an argument (KEY=VALUE)." >&2; usage; fi key=$(echo "$2" | cut -d= -f1 | tr '[:upper:]' '[:lower:]') @@ -105,13 +115,16 @@ template_file="typst-${doc_type}.typ" echo "Info: Using document type: $doc_type (template: $template_file)" >&2 # --- Determine Output Path --- -output_arg="" +# Reverted: Simplified output path logic +output_arg="" # Will be '-o path' or '-' for stdout + if [ "$output_file" = "-" ]; then - echo "Info: Outputting to stdout." >&2 + echo "Info: Outputting PDF to stdout." >&2 output_arg="-" - use_override_pipeline=true # Stdout requires the typst compile step + # PDF to stdout requires the typst compile step, even if no other overrides exist + use_override_pipeline=true elif [ -n "$output_file" ]; then - # User specified exact output file + # User specified exact output file path relative to output_dir output_path="${output_dir}/${output_file}" output_arg="-o $output_path" else @@ -121,16 +134,18 @@ else usage fi base_name=$(basename "$input_file" .md) - output_path="${output_dir}/${base_name}.pdf" + output_path="${output_dir}/${base_name}.pdf" # Always .pdf now output_arg="-o $output_path" fi # Create output directory if needed and not outputting to stdout if [ "$output_arg" != "-" ]; then mkdir -p "$(dirname "$output_path")" - echo "Info: Output path: $output_path" >&2 + echo "Info: Output PDF path: $output_path" >&2 fi +# Reverted: Removed adjustment logic based on output_format + # --- Build Commands --- # Check if PANDOC_DATA_DIR is set, provide default if not (useful if run outside Docker) @@ -138,16 +153,24 @@ PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-/usr/share/pandoc}" pandoc_base="pandoc --data-dir=\"$PANDOC_DATA_DIR\" --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" # --- Execute Pipeline --- +# Reverted: Simplified execution logic back to original override check echo "Info: Starting build..." >&2 if [ "$use_override_pipeline" = true ]; then - echo "Info: Using override pipeline (Pandoc -> Typst)." >&2 - # Need eval to handle spaces in arguments correctly, especially $typst_input_args and paths - eval "$pandoc_base --to=typst --template=\"$template_file\" \"$input_arg\" | typst compile $typst_input_args - $output_arg" + echo "Info: Using override pipeline (Pandoc -> Typst -> PDF)." >&2 + # Construct the pandoc part of the pipe + pandoc_cmd_part="$pandoc_base --to=typst --template=$template_file $input_arg" + # Execute pandoc part directly and pipe to typst compile + $pandoc_cmd_part | typst compile $typst_input_args - $output_arg else echo "Info: Using direct pipeline (Pandoc -> PDF)." >&2 - # Need eval for paths - eval "$pandoc_base --template=\"$template_file\" \"$input_arg\" $output_arg" + # Construct the full pandoc command parts + pandoc_cmd_part1="$pandoc_base --template=$template_file" + pandoc_cmd_part2="$input_arg" + # $output_arg contains '-o path' or is empty if default CWD output + # Execute directly, let shell handle splitting of $output_arg + $pandoc_cmd_part1 "$pandoc_cmd_part2" $output_arg fi -echo "Info: Build finished successfully." >&2 + +echo "Info: Build finished." >&2 exit 0 From ed779c6e265455dfc3f5864b7421945356e23638 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:30:40 +0000 Subject: [PATCH 09/63] Add Bats tests for build.sh functionality and submodule configuration --- .gitmodules | 9 +++++ tests/unit/build_sh.bats | 85 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 .gitmodules create mode 100755 tests/unit/build_sh.bats diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..28655fa --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "tests/bats"] + path = tests/bats + url = https://github.com/bats-core/bats-core.git@v1.11.1 +[submodule "tests/test_helper/bats-support"] + path = tests/test_helper/bats-support + url = https://github.com/bats-core/bats-support.git@v0.3.0 +[submodule "tests/test_helper/bats-assert"] + path = tests/test_helper/bats-assert + url = https://github.com/bats-core/bats-assert.git@v2.1.0 diff --git a/tests/unit/build_sh.bats b/tests/unit/build_sh.bats new file mode 100755 index 0000000..d9126b6 --- /dev/null +++ b/tests/unit/build_sh.bats @@ -0,0 +1,85 @@ +#!/usr/bin/env bats + +# Load Bats helpers from submodules +load '../test_helper/bats-support/load.bash' +load '../test_helper/bats-assert/load.bash' + +setup() { + # Create a dummy input file for tests that need one + mkdir -p "$BATS_TMPDIR/fixtures" + cat > "$BATS_TMPDIR/fixtures/dummy.md" < Date: Sat, 26 Apr 2025 10:32:11 +0000 Subject: [PATCH 10/63] Add tests and example files for Typst CV and linkify filters --- tests/filter/filters.bats | 108 +++++++++++++++++++++++++ tests/fixtures/example-cv.md | 78 ++++++++++++++++++ tests/fixtures/example-letter.md | 35 ++++++++ tests/fixtures/linkify_test.md | 13 +++ tests/fixtures/placeholder-photo.png | Bin 0 -> 95 bytes tests/fixtures/typst_cv_filter_test.md | 30 +++++++ 6 files changed, 264 insertions(+) create mode 100755 tests/filter/filters.bats create mode 100644 tests/fixtures/example-cv.md create mode 100644 tests/fixtures/example-letter.md create mode 100644 tests/fixtures/linkify_test.md create mode 100644 tests/fixtures/placeholder-photo.png create mode 100644 tests/fixtures/typst_cv_filter_test.md diff --git a/tests/filter/filters.bats b/tests/filter/filters.bats new file mode 100755 index 0000000..80b3bc2 --- /dev/null +++ b/tests/filter/filters.bats @@ -0,0 +1,108 @@ +#!/usr/bin/env bats + +# Load Bats helpers from submodules +load '../test_helper/bats-support/load.bash' +load '../test_helper/bats-assert/load.bash' + +# --- Configuration --- +FIXTURES_DIR="./tests/fixtures" +# Assuming PANDOC_DATA_DIR is set correctly in the environment or using default +PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-/usr/share/pandoc}" +# Base pandoc command parts - use eval later if needed for complex args +PANDOC_BASE_CMD="pandoc --data-dir=\"$PANDOC_DATA_DIR\" --to=typst" + +setup() { + # Determine project root relative to this test file's directory + PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + # Ensure fixtures are referenced from project root if needed, or adjust paths + FIXTURES_DIR="$PROJECT_ROOT/tests/fixtures" +} + +# --- linkify.lua Tests --- + +@test "filter/linkify: links Typst" { + local fixture="$FIXTURES_DIR/linkify_test.md" + local cmd="$PANDOC_BASE_CMD --lua-filter=linkify.lua \"$fixture\"" + run eval "$cmd" # Use eval as cmd string contains quotes + assert_success + assert_output --partial '#link("https://typst.app/")[Typst]' +} + +@test "filter/linkify: links Pandoc" { + local fixture="$FIXTURES_DIR/linkify_test.md" + local cmd="$PANDOC_BASE_CMD --lua-filter=linkify.lua \"$fixture\"" + run eval "$cmd" + assert_success + assert_output --partial '#link("https://pandoc.org/")[Pandoc]' +} + +@test "filter/linkify: links GitHub" { + local fixture="$FIXTURES_DIR/linkify_test.md" + local cmd="$PANDOC_BASE_CMD --lua-filter=linkify.lua \"$fixture\"" + run eval "$cmd" + assert_success + assert_output --partial '#link("https://github.com/")[GitHub]' +} + +@test "filter/linkify: does not link words not in YAML" { + local fixture="$FIXTURES_DIR/linkify_test.md" + local cmd="$PANDOC_BASE_CMD --lua-filter=linkify.lua \"$fixture\"" + run eval "$cmd" + assert_success + refute_output --partial '#link.*filter' # Use regex check here +} + + +# --- typst-cv.lua Tests --- + +@test "filter/typst-cv: handles photo attribute" { + local fixture="$FIXTURES_DIR/typst_cv_filter_test.md" + local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" + run eval "$cmd" + assert_success + assert_output --partial 'side: profile-photo(image("photo.png", width: 100pt))' +} + +@test "filter/typst-cv: handles hidden class" { + local fixture="$FIXTURES_DIR/typst_cv_filter_test.md" + local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" + run eval "$cmd" + assert_success + assert_output --partial '#hidden-heading()' +} + +@test "filter/typst-cv: handles location attribute" { + local fixture="$FIXTURES_DIR/typst_cv_filter_test.md" + local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" + run eval "$cmd" + assert_success + # Check for the specific function call and the content (actual output has single backslash) + assert_output --partial 'side: event-date()[City \ Country]' +} + +@test "filter/typst-cv: handles date attribute" { + local fixture="$FIXTURES_DIR/typst_cv_filter_test.md" + local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" + run eval "$cmd" + assert_success + assert_output --partial 'side: company-location()[2023 - Present]' +} + +@test "filter/typst-cv: handles ordered list" { + local fixture="$FIXTURES_DIR/typst_cv_filter_test.md" + local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" + run eval "$cmd" + assert_success + assert_output --partial '#block(width: full-width)' +} + +@test "filter/typst-cv: handles header with both date and location (processes date)" { + local fixture="$FIXTURES_DIR/typst_cv_filter_test.md" + local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" + run eval "$cmd" + assert_success + # Check date was processed (include closing parenthesis) + assert_output --partial 'side: company-location()[Date First])' + # Check location was NOT processed on the same header + refute_output --partial 'Header With Both.*side: event-date()' # Keep .* here for refute flexibility +} diff --git a/tests/fixtures/example-cv.md b/tests/fixtures/example-cv.md new file mode 100644 index 0000000..5f9f4fa --- /dev/null +++ b/tests/fixtures/example-cv.md @@ -0,0 +1,78 @@ +--- +title: Curriculum Vitae +author: Jane Doe +date: # Use structured date + year: 2025 + month: 4 + day: 25 +email: jane.doe@example.com +phone: +1-555-123-4567 +website: example.com +github: janedoe +linkedin: janedoe +# Optional: Define links used in the document +links: + website: https://example.com + Typst: https://typst.app/ + Pandoc: https://pandoc.org/ +# Optional: Add private data here, can be overridden by env vars or --set +private_info: This is private info from YAML +--- + +# Jane Doe {photo='image("tests/fixtures/placeholder-photo.png", width: 120pt)'} + +## Summary + +A highly motivated and skilled professional seeking a challenging position. Experienced in various technologies and methodologies. Eager to contribute to innovative projects. Check my website. + +## Experience {.hidden} + +### Example Corp {location="Remote"} +*Leading provider of innovative examples* + +#### Senior Example Engineer {date="2022 - Present"} +- Developed and maintained key features for the company's flagship product using advanced placeholder techniques. +- Collaborated with cross-functional teams (Product, Design, QA) to deliver high-quality software releases on schedule. +- Mentored junior engineers, conducted code reviews, and improved team coding standards. + +### Another Company Inc. {location="Example City"} +*Pioneers in fictional services* + +#### Example Intern {date="Summer 2021"} +- Assisted the development team with various tasks including testing and documentation. +- Gained practical experience in the software development lifecycle and agile methodologies. + +## Education {.hidden} + +### University of Example {location="Example City, EX"} + +#### M.S. in Computer Science {date="2020 - 2022"} +*Thesis: Advanced Topics in Placeholders* +Relevant coursework: Advanced Algorithms, Distributed Systems, Machine Learning. + +### Example College {location="Another Town, AT"} + +#### B.S. in Software Engineering {date="2016 - 2020"} +*Minor: Fictional Studies* +Graduated with honors. Capstone project involved developing a sample application. + +## Skills + +- **Programming:** Python, JavaScript, Typst, Shell Scripting +- **Tools:** Git, Docker, Pandoc +- **Concepts:** Agile Development, Test-Driven Development + +## Projects + +### Typst CV Generator + +A project to generate CVs and letters using Typst. + +### Placeholder Project + +Another example project demonstrating skills. + +## Languages + +- English (Native) +- French (Conversational) diff --git a/tests/fixtures/example-letter.md b/tests/fixtures/example-letter.md new file mode 100644 index 0000000..b1a42f8 --- /dev/null +++ b/tests/fixtures/example-letter.md @@ -0,0 +1,35 @@ +--- +title: Cover Letter +author: Jane Doe +date: April 24, 2025 +recipient: | + Hiring Manager + Example Corp + 123 Example St + Example City, EX 12345 +email: jane.doe@example.com +phone: +1-555-123-4567 +website: example.com +github: janedoe +linkedin: janedoe +# Optional: Define links used in the document +links: + typst: https://typst.app/ + Pandoc: https://pandoc.org/ +# Optional: Add private data here, can be overridden by env vars or --set +private_info: This is private info from YAML +--- + +Dear Hiring Manager, + +I am writing to express my strong interest in the Example Role position advertised on [Example.com](https://example.com/jobs/123). With my background in developing innovative solutions and my experience with technologies relevant to your stack, I am confident I possess the skills and qualifications necessary to make a significant contribution to Example Corp. + +In my previous role at Another Company Inc., I was responsible for [mention a key achievement or responsibility]. This experience allowed me to hone my skills in [mention relevant skill 1] and [mention relevant skill 2], both of which are mentioned as requirements in the job description. I am particularly drawn to Example Corp's work in [mention company area of interest] and believe my proactive approach and collaborative spirit would be a great asset to your team. + +I am proficient in various tools and technologies, including those used in this project like Pandoc and Typst. I am eager to learn more about this opportunity and discuss how my qualifications align with your needs. + +Thank you for your time and consideration. I have attached my CV for your review and look forward to hearing from you soon. + +Sincerely, + +Jane Doe diff --git a/tests/fixtures/linkify_test.md b/tests/fixtures/linkify_test.md new file mode 100644 index 0000000..11d97c7 --- /dev/null +++ b/tests/fixtures/linkify_test.md @@ -0,0 +1,13 @@ +--- +title: Linkify Test +links: + Typst: https://typst.app/ + Pandoc: https://pandoc.org/ + GitHub: https://github.com/ +--- + +This document tests the linkify filter. + +It should link Typst and Pandoc automatically. +Words like GitHub should also be linked. +Words not in the list, like filter, should remain plain text. diff --git a/tests/fixtures/placeholder-photo.png b/tests/fixtures/placeholder-photo.png new file mode 100644 index 0000000000000000000000000000000000000000..1914264c08781d1f30ee0b8482bccf44586f2dc1 GIT binary patch literal 95 zcmeAS@N?(olHy`uVBq!ia0vp^j3CU&3?x-=hn)ga%mF?ju0VQumF+E%TuG2$FoVOh l8)-lem#2$k2*>s01R$Gz9%CSj!PC{xWt~$(697H@6ZHT9 literal 0 HcmV?d00001 diff --git a/tests/fixtures/typst_cv_filter_test.md b/tests/fixtures/typst_cv_filter_test.md new file mode 100644 index 0000000..43b4d5a --- /dev/null +++ b/tests/fixtures/typst_cv_filter_test.md @@ -0,0 +1,30 @@ +--- +title: Typst CV Filter Test - Current Logic +--- + +# Name {photo='image("photo.png", width: 100pt)'} + +This header has a photo attribute. + +## Section 1 {.hidden} + +This header has the hidden class. + +### Company {location="City \\ Country"} + +This header has a location attribute. + +#### Position {date="2023 - Present"} + +This header has a date attribute. + +##### Unchanged Header + +This header has no special attributes or classes. + +###### Header With Both {date="Date First" location="Location Second"} + +This header has both date and location. The filter should only process the first one it finds (likely date). + +1. First item in ordered list. +2. Second item in ordered list. From 9079fe2bc077f1e33887263c2f0c7ee4bcf6a570 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:32:27 +0000 Subject: [PATCH 11/63] Add E2E smoke test script for PDF generation using build.sh --- tests/test_e2e.sh | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 tests/test_e2e.sh diff --git a/tests/test_e2e.sh b/tests/test_e2e.sh new file mode 100755 index 0000000..9677ffe --- /dev/null +++ b/tests/test_e2e.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -e + +# E2E smoke test script for build.sh (generating PDF) + +# --- Configuration --- +FIXTURES_DIR="./tests/fixtures" +BUILD_SCRIPT="./build.sh" +TEST_OUTPUT_DIR=$(mktemp -d) + +echo "--- Running E2E Smoke Tests (PDF Output) ---" + +cleanup() { + echo "Cleaning up temporary directory: $TEST_OUTPUT_DIR" + rm -rf "$TEST_OUTPUT_DIR" +} +trap cleanup EXIT # Ensure cleanup happens on script exit + +# --- Test example-cv.md PDF Generation --- +echo "Testing example-cv.md -> PDF..." +CV_FIXTURE="${FIXTURES_DIR}/example-cv.md" +CV_EXPECTED_FILENAME="example-cv.pdf" +CV_EXPECTED_PDF="${TEST_OUTPUT_DIR}/${CV_EXPECTED_FILENAME}" + +# Run build.sh to generate PDF output, specifying output dir and filename +"$BUILD_SCRIPT" --output-dir "$TEST_OUTPUT_DIR" --output "$CV_EXPECTED_FILENAME" "$CV_FIXTURE" +exit_code=$? + +if [ $exit_code -ne 0 ]; then + echo "Error: build.sh failed for example-cv.md PDF generation" >&2 + exit 1 +fi + +# Check if PDF exists and has size > 0 +if [ ! -f "$CV_EXPECTED_PDF" ]; then + echo "Error: Expected PDF file not found: $CV_EXPECTED_PDF" >&2 + exit 1 +fi +if [ ! -s "$CV_EXPECTED_PDF" ]; then + echo "Error: Generated PDF file is empty: $CV_EXPECTED_PDF" >&2 + exit 1 +fi +echo "example-cv.md PDF generated successfully." + +# --- Test example-letter.md PDF Generation --- +echo "Testing example-letter.md -> PDF..." +LETTER_FIXTURE="${FIXTURES_DIR}/example-letter.md" +LETTER_EXPECTED_FILENAME="example-letter.pdf" +LETTER_EXPECTED_PDF="${TEST_OUTPUT_DIR}/${LETTER_EXPECTED_FILENAME}" + +# Run build.sh to generate PDF output, specifying output dir and filename +"$BUILD_SCRIPT" --output-dir "$TEST_OUTPUT_DIR" --output "$LETTER_EXPECTED_FILENAME" "$LETTER_FIXTURE" --type letter +exit_code=$? + +if [ $exit_code -ne 0 ]; then + echo "Error: build.sh failed for example-letter.md PDF generation" >&2 + exit 1 +fi + +# Check if PDF exists and has size > 0 +if [ ! -f "$LETTER_EXPECTED_PDF" ]; then + echo "Error: Expected PDF file not found: $LETTER_EXPECTED_PDF" >&2 + exit 1 +fi +if [ ! -s "$LETTER_EXPECTED_PDF" ]; then + echo "Error: Generated PDF file is empty: $LETTER_EXPECTED_PDF" >&2 + exit 1 +fi +echo "example-letter.md PDF generated successfully." + + +echo "--- E2E Smoke Tests Passed ---" +exit 0 From e2aeaf8b021cf28c41c366a4b71943112eaeef7f Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:32:56 +0000 Subject: [PATCH 12/63] Refactor CI workflow: update artifact handling, remove publish job, and enhance test execution --- .github/workflows/ci.yml | 106 +++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43c9db4..ce293ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,9 +11,9 @@ on: env: IMAGE: ${{ vars.DOCKERHUB_USERNAME }}/${{ vars.DOCKERHUB_REPOSITORY }} IMAGE_TAG_TESTING: ${{ vars.DOCKERHUB_USERNAME }}/${{ vars.DOCKERHUB_REPOSITORY }}:testing - OUTPUT_DIR: public OUTPUT_ARTIFACTS_NAME: pdf - SOURCE_ARTIFACTS_NAME: source + EXAMPLE_ARTIFACTS_NAME: examples + TEST_DIR: tests # Directory containing test scripts and fixtures jobs: pre_commit: @@ -38,16 +38,15 @@ jobs: - name: Upload source files uses: actions/upload-artifact@v4 with: - name: ${{ env.SOURCE_ARTIFACTS_NAME }} + name: ${{ env.EXAMPLE_ARTIFACTS_NAME }} path: | - ./kadykov-*.md - ./photo.jpg - ./justfile + ${{ env.TEST_DIR }}/fixtures/example-*.md + ${{ env.TEST_DIR }}/fixtures/placeholder-photo.png if-no-files-found: error docker: - name: Build and push testing Docker image - needs: source + name: Build, test, and push Docker image + needs: pre_commit # Doesn't need source artifact anymore, tests are in repo runs-on: ubuntu-latest steps: - name: Set up Docker Buildx @@ -64,29 +63,50 @@ jobs: with: load: true tags: ${{ env.IMAGE_TAG_TESTING }} - cache-from: type=registry,ref=${{ env.IMAGE }}:${{ github.ref_name }} + cache-from: type=registry,ref=${{ env.IMAGE }}:testing # Cache from previous testing tag if exists cache-to: type=inline - - name: Download source files - uses: actions/download-artifact@v4 - with: - name: ${{ env.SOURCE_ARTIFACTS_NAME }} - - - name: Render with testing Docker image - run: > - docker container run - -v "${PWD}:/data" - --user "$(id -u):$(id -g)" - ${{ env.IMAGE_TAG_TESTING }} - - - name: Upload rendered PDFs + - name: Run tests inside the container + run: | + echo "Running tests..." + # Make scripts executable + docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + sh -c "chmod +x ${{ env.TEST_DIR }}/*.sh" + + # Execute test suites + docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + bats ${{ env.TEST_DIR }}/unit/build_sh.bats + + docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + bats ${{ env.TEST_DIR }}/filter/filters.bats # Use bats for filter tests + + # Removed execution of test_integration.sh + # docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + # sh ${{ env.TEST_DIR }}/test_integration.sh + + docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + sh ${{ env.TEST_DIR }}/test_e2e.sh + echo "Tests finished." + + - name: Build Example PDFs for Release Artifacts (after tests pass) + run: | + echo "Building example PDFs..." + docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + ./build.sh --output-dir examples ${{ env.TEST_DIR }}/fixtures/example-cv.md + docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + ./build.sh --output-dir examples ${{ env.TEST_DIR }}/fixtures/example-letter.md --type letter + echo "Example PDFs built." + + - name: Upload Example PDFs uses: actions/upload-artifact@v4 with: - name: ${{ env.OUTPUT_ARTIFACTS_NAME }} - path: ./*.pdf + name: ${{ env.OUTPUT_ARTIFACTS_NAME }} # Use the same name expected by release job + path: ./examples/*.pdf if-no-files-found: error - - name: Login to Docker Hub + # Only push final images if tests passed and it's a push to main or a tag + - name: Login to Docker Hub (for push) + if: success() && (github.event_name == 'push') # Only push on push events uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} @@ -108,34 +128,10 @@ jobs: with: push: true tags: ${{ steps.meta.outputs.tags }} - cache-from: type=registry,ref=${{ env.IMAGE }}:${{ github.ref_name }} + cache-from: type=registry,ref=${{ env.IMAGE }}:testing # Use testing cache cache-to: type=inline - publish: - name: Publish to GitHub pages - needs: docker - runs-on: ubuntu-latest - permissions: - contents: write - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - steps: - - uses: actions/checkout@v4 - - - name: Download rendered PDFs - uses: actions/download-artifact@v4 - with: - name: ${{ env.OUTPUT_ARTIFACTS_NAME }} - path: ${{ env.OUTPUT_DIR }}/ - - - name: Add HTML for GitHub Pages - run: cp ./*.html ${{ env.OUTPUT_DIR }}/ - - - name: Deploy CV to GitHub pages - uses: peaceiris/actions-gh-pages@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./${{ env.OUTPUT_DIR }} + # Removed the 'publish' job for GitHub Pages CV deployment release: name: Create a GitHub release @@ -153,11 +149,11 @@ jobs: run: echo "current_version=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_OUTPUT" shell: bash - - name: Download rendered PDFs + - name: Download Example PDFs uses: actions/download-artifact@v4 with: - name: ${{ env.OUTPUT_ARTIFACTS_NAME }} - path: ${{ env.OUTPUT_DIR }}/ + name: ${{ env.OUTPUT_ARTIFACTS_NAME }} # Artifact containing example-cv.pdf, example-letter.pdf + path: ./release-assets/ # Download to a specific directory - name: Get Changelog Entry id: changelog_reader @@ -177,4 +173,4 @@ jobs: draft: ${{ steps.changelog_reader.outputs.status == 'unreleased' }} allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} - artifacts: ${{ env.OUTPUT_DIR }}/*.pdf + artifacts: ./release-assets/*.pdf # Upload PDFs from the download directory From 6c9a0808591dba87735059256bd697199c059031 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:33:13 +0000 Subject: [PATCH 13/63] Add Justfile for common development tasks and testing automation --- justfile | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 justfile diff --git a/justfile b/justfile new file mode 100644 index 0000000..12ce525 --- /dev/null +++ b/justfile @@ -0,0 +1,41 @@ +# Justfile for common development tasks + +# Variables +bats_executable := "tests/bats/bin/bats" +unit_tests_file := "tests/unit/build_sh.bats" +filter_tests_file := "tests/filter/filters.bats" # Updated path +e2e_tests_script := "tests/test_e2e.sh" + +# Default task +default: + @just --list + +# Run all linters/formatters +lint: + pre-commit run --all-files + +# Run unit tests +test-unit: + {{bats_executable}} {{unit_tests_file}} + +# Run filter tests (Pandoc Lua filters) +test-filter: + {{bats_executable}} {{filter_tests_file}} # Use bats now + +# Run E2E smoke tests (PDF generation) +test-e2e: + sh {{e2e_tests_script}} + +# Run all tests +test: test-unit test-filter test-e2e + @echo "All tests passed!" + +# Clean temporary files (if any - currently handled by tests) +clean: + @echo "No specific clean actions defined." + +# Build example files (useful for quick checks) +build-examples: + ./build.sh --output-dir examples tests/fixtures/example-cv.md + ./build.sh --output-dir examples tests/fixtures/example-letter.md --type letter + @echo "Example PDFs built in ./examples/" From 9234cfc18b44c459d4b56f076deb4a6af663a6b2 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:33:43 +0000 Subject: [PATCH 14/63] Enhance README: clarify priority of overrides and recommended workflow for private data, update examples for better structure and clarity --- README.md | 99 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 5225f38..2e167fd 100644 --- a/README.md +++ b/README.md @@ -126,9 +126,9 @@ You can override values defined in the YAML frontmatter without editing the Mark ./build.sh my-cv.md --set email=private@email.com --set phone="+1 123 456 7890" --set title="Senior Engineer Application" ``` -**Priority:** Values from `--set` arguments and `TYPSTCV_` environment variables take precedence over values defined in the YAML frontmatter. The Typst template checks for these overrides first. +**Priority & Workflow:** Values from `--set` arguments and `TYPSTCV_` environment variables take precedence over values defined in the YAML frontmatter. The Typst template checks for these overrides first. The recommended workflow is to keep only public, shareable information in your committed Markdown/YAML file and use overrides to supply private details (like phone numbers) or make temporary modifications for specific builds. -**Note:** The `phone` field is typically *only* provided via overrides, as it's usually considered private information not included directly in the source Markdown/YAML. +**Note:** Fields like `phone` are typically *only* provided via overrides, as they are usually considered private information not suitable for version control. --- @@ -148,35 +148,42 @@ Any occurrence of "TDS" or "THz" in the Markdown body will be linked accordingly ### Right-Side Content (CVs) -You can display content -(e.g., profile picture, location, dates) -on the right side of your CV. -To enable add metadata (`key="value"`) -to the heading you want to associate with side content. +You can display content (e.g., profile picture, location, dates) on the right side of your CV headings using Markdown attributes. The `typst-cv.lua` filter translates these attributes into specific Typst layout functions. -**Supported keys:** +**Recommended Structure & Supported Attributes:** -- `photo`: Typst block with your profile image. -- `location`: Typst block with location details. -- `date`: Typst block with dates. +- **`# Level 1 Heading {photo='...'}`:** Typically used for the main title/name with a profile photo. The value should be a Typst `image()` call (e.g., `{photo='image("./placeholder-photo.png", width: 120pt)'}`). +- **`### Level 3 Heading {location="..."}`:** Recommended for Company or Institution names. The value is the location text. +- **`#### Level 4 Heading {date="..."}`:** Recommended for Position or Degree titles. The value is the date range text. + +**Important Notes:** + +- The filter processes only the *first* supported attribute (`photo`, `location`, or `date`) it finds on a heading. Do not combine them on the same heading. +- Use `\\` for line breaks within attribute values if needed (e.g., `{location="City \\ Country"}`). Pandoc requires double backslashes for a literal newline in Typst output. **Example:** ```markdown -## Multitel ASBL {location="Mons \\ Belgium"} +# Jane Doe {photo='image("./placeholder-photo.png", width: 120pt)'} -_Non-profit innovation center_ +## Experience {.hidden} -### Research Engineer {date="Jul.~2021 \\ Aug.~2024"} +### Example Corp {location="Remote"} +*Leading provider of innovative examples* -Developed a THz time-domain spectroscopy (THz-TDS) data pipeline with an improved signal-to-noise ratio using sensitivity profile-shaped filtering. -``` +#### Senior Example Engineer {date="2022 - Present"} +- Developed key features... -**Note:** Use `\\` for line breaks in Typst blocks (Pandoc strips single `\`). +### Another Company Inc. {location="Example City"} +*Pioneers in fictional services* + +#### Example Intern {date="Summer 2021"} +- Assisted the team... +``` #### Known Issue -Side content automatically starts a new paragraph after the heading. To avoid extra spacing, structure your content accordingly. +Side content might visually appear slightly detached or start a new paragraph relative to the heading text depending on Typst's layout decisions. Structure your content accordingly. --- @@ -212,49 +219,45 @@ and they automatically span the full width of the document. ### CV Example -Below is a minimal example of a CV in Markdown. -For a detailed example, see [`kadykov-cv-en.md`](kadykov-cv-en.md). -Rendered example PDFs are available [here](http://github.kadykov.com/typstCV/). +Below is a minimal example of a CV in Markdown, demonstrating the recommended structure. +For a runnable example, see [`tests/fixtures/example-cv.md`](tests/fixtures/example-cv.md). ```markdown --- -author: Aleksandr KADYKOV -title: Research Engineer -email: cv@kadykov.com -github: kadykov -gitlab: kadykov -linkedin: aleksandr-kadykov -website: www.kadykov.com -keywords: - - résumé - - resume - - CV +author: Jane Doe +title: Curriculum Vitae +email: jane.doe@example.com +github: janedoe +linkedin: janedoe +website: example.com +# YAML contains public data. Private data (e.g., phone) is added via overrides. links: - TDS: https://en.wikipedia.org/wiki/Terahertz_time-domain_spectroscopy - THz: https://en.wikipedia.org/wiki/Terahertz_radiation - + website: https://example.com + Typst: https://typst.app/ + Pandoc: https://pandoc.org/ --- -# Research Engineer {photo='image("./photo.jpg", width: 120pt)'} +# Jane Doe {photo='image("./placeholder-photo.png", width: 120pt)'} -Headline of your CV +A highly motivated professional... Check my website. -# Core Competencies {.hidden} +## Experience {.hidden} -- Data analysis & presentation -- Experimental design & execution -- Instrumentation integration & orchestration -- Scientific Python development +### Example Corp {location="Remote"} +*Leading provider of innovative examples* -# Professional Experience {.hidden} +#### Senior Example Engineer {date="2022 - Present"} +- Developed key features... ---- +## Education {.hidden} -## Multitel ASBL {location="Mons \\ Belgium"} +### University of Example {location="Example City, EX"} -_Company description._ +#### M.S. in Computer Science {date="2020 - 2022"} +*Thesis: Advanced Topics in Placeholders* -### Research Engineer {date="Jul.~2021 \\ Aug.~2024"} +## Skills {.hidden} -Your explanation of what you have done. +- Programming: Python, JavaScript, Typst, Shell Scripting +- Tools: Git, Docker, Pandoc ``` From 0ee0e1965069926d2e9c07a69eaa7ccecc399d4c Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:34:00 +0000 Subject: [PATCH 15/63] Update activeContext, progress, and techContext documentation: reflect completion of Phase 1, enhance testing details, and clarify development environment setup --- memory-bank/activeContext.md | 39 ++++++++++++++-------- memory-bank/progress.md | 65 ++++++++++++++++++++---------------- memory-bank/techContext.md | 14 +++++--- 3 files changed, 70 insertions(+), 48 deletions(-) diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index e74b1ac..d609872 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,24 +1,35 @@ -# Active Context: Interface Refactoring Complete (2025-04-23) +# Active Context: Testing & Depersonalization Complete (2025-04-26) ## Current Focus -The refactoring of the project's user interface is complete. The `justfile` has been replaced with `build.sh`, providing a flexible command-line interface with support for overrides and stdin/stdout. Docker integration has been updated accordingly. +Phase 1, focusing on adding tests, depersonalizing examples, and updating CI, is complete. The project now has a test suite covering unit, filter, and E2E scenarios, uses generic example files, and the CI workflow automatically runs these tests. ## Recent Actions (Completed) -- Reviewed project structure, `README.md`, `justfile`, Lua filters (`linkify.lua`, `typst-cv.lua`), Typst templates (`typst-cv.typ`, `typst-letter.typ`), style package (`style.typ`, `typst.toml`), `Dockerfile`, and `entrypoint.sh`. -- Confirmed the rendering pipeline (Markdown -> Pandoc w/ Filters & Template -> Typst -> PDF). -- Identified the `justfile`'s hardcoded values and rigid structure as the main usability bottleneck. -- Analyzed the mechanism for handling private data (`.env` + `just build-private` piping to `typst compile --input`) and identified it as inflexible. -- Examined Docker setup and identified the need to update the `ENTRYPOINT`. +- **Discussed Next Steps:** Prioritized adding tests and depersonalizing the project. +- **Test Setup:** + - Created `tests/` directory structure. + - Installed `bats-core` and helpers via Git submodules (`tests/bats`, `tests/test_helper`). +- **Depersonalization:** + - Created generic example files (`tests/fixtures/example-cv.md`, `tests/fixtures/example-letter.md`) using placeholder data and correct heading/link structure. + - Created placeholder image (`tests/fixtures/placeholder-photo.png`). + - Removed old personalized files (`kadykov-*.md`, `photo.jpg`). +- **Testing Implementation:** + - Added unit tests for `build.sh` using Bats (`tests/unit/build_sh.bats`), debugging path and execution issues. + - Added filter tests (`tests/test_filters.sh`) comparing Pandoc+filter output against snapshots (using templates where necessary). Updated snapshots to match actual Pandoc output formatting. + - Added E2E smoke tests (`tests/test_e2e.sh`) verifying successful PDF generation for example files. Debugged issues related to YAML date format and image paths in templates/fixtures. +- **Documentation:** Updated `README.md` to clarify heading structure, linkification, metadata override workflow, and point to new example files. +- **CI Update:** Modified `.github/workflows/ci.yml` to run the new test suites (unit, filter, e2e) inside the Docker container and removed the old GitHub Pages deployment job for CVs. Ensured example PDFs are built and uploaded for release artifacts. +- **Development Tooling:** Added `justfile` for convenient local execution of linters and test suites. +- **Memory Bank:** Updated `activeContext.md`, `progress.md`, `techContext.md`. -1. **Modified `style.typ`:** Implemented universal override logic (checking `sys.inputs` first) and renamed `public-email` to `email`. -2. **Created `build.sh`:** Implemented the new flexible command-line interface with argument parsing, stdin/stdout support, override detection (`TYPSTCV_*`, `--set`), and dynamic pipeline selection. -3. **Updated `Dockerfile`:** Changed `ENTRYPOINT` to `build.sh` and added steps to copy/chmod the script. -4. **Updated Documentation:** Modified `README.md` with new usage instructions for `build.sh` (CLI & Docker, including correct Docker Hub tag `kadykov/typst-cv`) and updated example files (`kadykov-*.md`) to use `email`. -5. **Cleaned Up:** Removed `justfile`, `action.yml`, and `entrypoint.sh`. -6. **Updated Memory Bank:** Updated `activeContext.md` and `progress.md`. +## Decisions & Notes + +- Decided *not* to implement `--output-format typst` in `build.sh` at this time due to difficulties in reliably generating/testing the `.typ` output via the script. Integration testing relies on filter tests and E2E PDF tests. +- Noted suggestions for future filter (`typst-cv.lua`) refactoring (fixing function names, adding generic side content, multiple attributes, div support) but postponed implementation until after establishing baseline tests. +- Corrected understanding of YAML date format and relative paths for Typst compilation. ## Immediate Next Steps -- Present the completed work to the user. +- Update `progress.md` and `techContext.md`. +- Present the completed Phase 1 work to the user. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 8eaa6e4..68e4f61 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -1,37 +1,44 @@ -# Progress: Typst CV/Letter Generator (As of 2025-04-23) +# Progress: Typst CV/Letter Generator (As of 2025-04-26) ## What Works (Current State) -- The core Markdown-to-PDF pipeline using Pandoc, Lua filters, and Typst functions correctly for the example files (`kadykov-*.md`). -- Rendering of both CVs and Letters works. -- Custom Markdown features (YAML metadata, `links`, attributes like `{photo}`, `{date}`, `{location}`, class `.hidden`) are implemented and functional via Lua filters and Typst styles. -- Public vs. Private rendering using `.env` and `just build`/`just build-private` works, albeit inflexibly. -- Docker image builds successfully and contains all necessary dependencies and project files. -- GitHub Action (`entrypoint.sh`) exists but relies on the current `justfile`. - -## What's Left to Build (Current Task) - -1. **`build.sh` Script:** Create the new command-line interface script. - - Argument parsing (input, output, type, overrides). - - Stdin/stdout support. - - Override detection (`TYPSTCV_*` env vars, `--set` args). - - Dynamic pipeline selection (Pandoc-direct-PDF vs. Pandoc-Typst | Typst-Compile). - * Command construction and execution. - * Error handling and user feedback. -2. **`Dockerfile` Update:** Modify the `ENTRYPOINT` to use `build.sh`. Copy `build.sh` into the image. -3. **`entrypoint.sh` Update:** Modify the GitHub Action script to use `build.sh` instead of `just build`. -4. **`README.md` Update:** Rewrite usage instructions for `build.sh` (CLI and Docker). -5. **`justfile` Removal:** Delete the old `justfile`. +- **Core Pipeline:** Markdown-to-PDF conversion using `build.sh`, Pandoc, Lua filters, and Typst functions correctly. +- **Interface:** Flexible `build.sh` CLI handles input/output options, type inference, stdin/stdout (for PDF), and metadata overrides (`--set`, `TYPSTCV_*`). +- **Features:** Automatic linkification (`linkify.lua`), CV side content (`{photo}`, `{location}`, `{date}` via `typst-cv.lua`), hidden headings (`.hidden`), and full-width ordered lists are functional. +- **Examples:** Generic, depersonalized example files (`example-cv.md`, `example-letter.md`) and placeholder image exist in `tests/fixtures/`. +- **Testing:** + - Unit tests (`bats`) for `build.sh` logic exist and pass. + - Filter tests (`sh`) comparing Pandoc output against snapshots exist and pass. + - E2E smoke tests (`sh`) verifying PDF generation for examples exist and pass. + - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.). +- **Docker:** Image builds, includes dependencies, uses `build.sh` as entrypoint. +- **CI:** GitHub Actions workflow lints code (`pre-commit`), builds Docker image, runs all test suites inside the container, builds example PDFs, and handles releases (uploads example PDFs, uses changelog). + +## What's Left to Build (Potential Future Work) + +- **Phase 2: User Experience & Flexibility:** + - Allow custom Pandoc Typst templates (beyond built-in `cv`/`letter`). + - Enhance style customization (more parameters in `style.typ` or allow custom style files). +- **Phase 3: Technical Challenges & Refinements:** + - Improve/Fix stdin/stdout support for Typst format output (overrides currently ignored). + - Refactor `typst-cv.lua` and `style.typ`: + - Fix swapped `event-date`/`company-location` function usage. + - Rename functions (e.g., `date()`, `location()`). + - Add generic `{side="..."}` attribute support. + - Support multiple attributes per heading. + - Add support for divs like `#aside`. + - Explore dynamic font installation (e.g., `fontist`). + - Separate Docker images for development (Debian/Fedora) and production (Alpine). +- **Phase 4: Rebranding & Marketing:** + - Rebrand project (name, repository, Docker Hub). + - Create GitHub Action for publishing documentation (e.g., to GitHub Pages). ## Current Status -- **Planning Phase Complete:** Analysis of the existing system is done. A detailed plan for refactoring the interface with `build.sh` has been agreed upon. -- **Memory Bank Established:** Initial versions of all core Memory Bank files have been created. -- **Ready for Implementation:** The next step is to draft the `build.sh` script. +- **Phase 1 Complete:** Testing framework established, examples depersonalized, CI updated, `justfile` added. Core functionality is tested. +- **Ready for Next Phase:** Project is in a stable state, ready for planning and implementation of new features or refactoring based on user priorities. -## Known Issues (Existing System) +## Known Issues -- **Rigid Interface:** `justfile` requires specific filenames (`kadykov-*`) and structure. -- **Inflexible Private Data:** `.env` + `build-private` is the only way to include private info. -- **Docker Entrypoint:** Tied to the inflexible `justfile`. -- **Minor Layout Quirk:** README mentions side content (`{date}`, `{location}`) starting a new paragraph after the heading, potentially causing extra space. (This is a Typst/styling detail, not directly addressed by the current refactoring task, but noted). +- **Minor Layout Quirk:** README mentions side content (`{date}`, `{location}`) might cause minor spacing variations depending on Typst layout. (Low priority Typst/styling detail). +- **Overrides Ignored for Typst Stdout:** `--set`/`TYPSTCV_*` overrides are currently ignored if outputting Typst format *to stdout* due to limitations in the pipeline. diff --git a/memory-bank/techContext.md b/memory-bank/techContext.md index 17c7986..a082a1c 100644 --- a/memory-bank/techContext.md +++ b/memory-bank/techContext.md @@ -9,7 +9,8 @@ - **YAML:** Used within Markdown frontmatter for metadata (`author`, `title`, `links`, etc.). - **Shell Script (Bash/sh):** Proposed for `build.sh` to replace `justfile`. Needs to be POSIX-compliant for broader compatibility (especially within minimal Docker containers like Alpine, though the current base is Fedora). - **Docker:** Containerization platform used for packaging and distribution. `Dockerfile` defines the image build process. -- **Just:** Task runner (currently used, to be replaced by `build.sh`). Version specified in `Dockerfile`. +- **Just:** Task runner used for local development tasks (linting, testing, building examples). Invoked via `just `. +- **Bats (Bash Automated Testing System):** Used for unit testing `build.sh`. Installed via Git submodule in `tests/bats`. Includes `bats-support` and `bats-assert` helpers. ## Key Files & Locations (within Docker image) @@ -26,11 +27,14 @@ - Specific versions of Fedora, Alpine (for builder stage), Pandoc, Just, Typst, and fonts are pinned in the `Dockerfile`. This ensures reproducibility. - External Typst packages (`@preview/fontawesome`) are also versioned. -## Build/Execution Environment +## Build/Execution/Testing Environment -- Primarily designed to run within the provided Docker container, which includes all necessary dependencies and configurations. -- The proposed `build.sh` script relies on standard POSIX shell commands and the installed tools (Pandoc, Typst). -- Environment variables (`TYPSTCV_*`, potentially `PANDOC_DATA_DIR`, `TYPST_PACKAGE_PATH`, `TYPST_FONT_PATHS`) play a role in configuration. +- Builds and final execution are designed to run within the provided Docker container (`kadykov/typst-cv`), which includes all necessary dependencies (Pandoc, Typst, Lua filters, Typst packages, fonts). +- Local development and testing utilize the devcontainer environment, which includes `bats-core`, `just`, `pre-commit`, and other development tools. +- The `build.sh` script relies on standard POSIX shell commands and the installed tools (Pandoc, Typst). +- Tests (`bats`, `sh`) are run from the project root. +- CI (GitHub Actions) runs linters and then executes the test suite inside the built Docker container. +- Environment variables (`TYPSTCV_*`, `PANDOC_DATA_DIR`, `TYPST_PACKAGE_PATH`, `TYPST_FONT_PATHS`) play a role in configuration, especially within Docker/CI. ## Constraints From 54e160c05e95aa0280d232c8b3e92ffe00e91386 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:34:45 +0000 Subject: [PATCH 16/63] Delete outdated CV and cover letter files in French and English. --- kadykov-cv-en.md | 471 ------------------------------------------- kadykov-cv-fr.md | 464 ------------------------------------------ kadykov-letter-en.md | 223 -------------------- kadykov-letter-fr.md | 227 --------------------- 4 files changed, 1385 deletions(-) delete mode 100644 kadykov-cv-en.md delete mode 100644 kadykov-cv-fr.md delete mode 100644 kadykov-letter-en.md delete mode 100644 kadykov-letter-fr.md diff --git a/kadykov-cv-en.md b/kadykov-cv-en.md deleted file mode 100644 index 53dcaff..0000000 --- a/kadykov-cv-en.md +++ /dev/null @@ -1,471 +0,0 @@ ---- -author: Aleksandr KADYKOV -title: Research Engineer -email: cv@kadykov.com -github: kadykov -gitlab: kadykov -linkedin: aleksandr-kadykov -website: www.kadykov.com -date: "year: 1970, month: 1, day: 1" -keywords: - - résumé - - resume - - CV - - Curriculum vitae - - Research Software Engineer - - Research Engineer - - Software Engineer - - THz - - terahertz - - THz-TDS - - spectroscopy - - cryogenics - - optics - - photonics - - measurements - - data analysis - - programming - - DevOps - - CI/CD - - TDD - - Test-Driven Development - - Python - - Jupyter - - NumPy - - Pandas - - Xarray - - Scipy - - Python array API - - scikit-learn - - PyTorch - - MATLAB - - Matplotlib - - hvPlot - - Plotly - - Bokeh - - Panel - - holoviz - - OriginPro - - PyMeasure - - Bluesky - - yaq - - LabVIEW - - Intake - - SQL - - Quarto - - Typst - - Pandoc - - LaTeX - - VSCode - - Git - - Linux - - Docker - - Docker-compose - - Zotero -links: - Multitel: https://www.multitel.eu/expertise/applied-photonics/terahertz-spectroscopy-and-imaging/ - ASBL: https://fr.wikipedia.org/wiki/Association_sans_but_lucratif - SAPHIRE: https://www.multitel.eu/projects/saphire/ - LNE: https://www.lne.fr/en/research-and-development - EPIC: https://en.wikipedia.org/wiki/%C3%89tablissement_public_%C3%A0_caract%C3%A8re_industriel_et_commercial - I2S: https://edi2s.umontpellier.fr/ - L2C: https://coulomb.umontpellier.fr/?lang=en - IPM: http://www.ipmras.ru/en/institute/scientific-departments/department-110/ - RAS: https://en.wikipedia.org/wiki/Russian_Academy_of_Sciences - TDS: https://en.wikipedia.org/wiki/Terahertz_time-domain_spectroscopy - THz: https://en.wikipedia.org/wiki/Terahertz_radiation - hBN: https://en.wikipedia.org/wiki/Boron_nitride - HgCdTe: https://en.wikipedia.org/wiki/Mercury_cadmium_telluride - graphene: https://en.wikipedia.org/wiki/Graphene - helium: https://en.wikipedia.org/wiki/Liquid_helium - Python: https://github.com/search?q=language%3APython+author%3Akadykov&type=pullrequests - wavelength: https://doi.org/10.1063/1.4996966 - FTIR: https://en.wikipedia.org/wiki/Fourier-transform_infrared_spectroscopy - FAIR: https://en.wikipedia.org/wiki/FAIR_data - Mons: https://www.openstreetmap.org/#map=19/50.45756/3.92540 - Trappes: https://www.openstreetmap.org/#map=17/48.76090/1.98370 - Nizhny: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Novgorod: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Thesis: https://www.theses.fr/en/2017MONTS086 ---- - -# Research Engineer {photo='image("./photo.jpg", width: 120pt)'} - -Proven ability to -design and execute experiments, -analyse and present data, -develop scientific Python software. - -Strong background in -applied and basic research -in THz photonics -and -magnetotransport -in 2D~materials. - -# Core competencies {.hidden} - -- Data analysis & presentation -- Experimental design & execution -- Instrumentation integration & orchestration -- Scientific Python development - -# Professional experience {.hidden} - ---- - -## Multitel ASBL {location="Mons \\ Belgium"} - -_Non-profit -innovation center specializing in -applied photonics, AI, etc._ - -### Research Engineer in THz Spectroscopy and Imaging {date="Jul.~2021 \\ Aug.~2024"} - -Developed -a THz time-domain spectroscopy (THz-TDS) -data pipeline -with an improved signal-to-noise ratio -by utilizing sensitivity profile-shaped filtering. - -Developed -a computationally cheap -THz-TDS data processing method -for refractive index and thickness extraction -in low-absorption materials. - -Streamlined -refractive index profile reconstruction -from THz-TDS data -by offloading calculations -to a GPU -and utilizing backpropagation-based -optimization algorithms. - - - -Automated laboratory workflows -by implementing Python tools -for measurement orchestration, -data management, -analysis, -and result presentation. - -Ensured best software development practices -by implementing unit testing, -CI/CD pipelines, -and documentation. - - - - - ---- - -## Laboratoire National de Métrologie et d'Essais {location="Trappes \\ France"} - -_French National Laboratory of Metrology and Testing (LNE)_ - -### Research Engineer in Quantum Hall Effect Metrology {date="Sep.~2018 \\ Sep.~2020"} - -Led low-noise -cryogenic -magnetotransport measurements -on graphene, -exploring its potential -as a resistance standard. - -Designed -a flexible Python software -package, - -optimizing scientific equipment orchestration. - -Participated -in the nanofabrication -of hBN-encapsulated graphene stacks. - -Improved performance -of a helium gas recuperation system. - ---- - -## Institute for Physics of Microstructures (IPM RAS) {location="Nizhny Novgorod \\ Russia"} - -_State-owned research institute -specializing in solid state physics._ - -### Research Engineer in Photonics of 2D Narrow-Gap Heterostructures {date="May~2017 \\ Sep.~2018"} - -Led -photoluminescence and photoconductivity -FTIR cryogenic measurements -of HgTe/HgCdTe quantum wells. - -Achieved -laser emission -in HgCdTe heterostructures at -a record wavelength. - ---- - -# Education {.hidden} - -## Laboratoire Charles Coulomb (L2C) & IPM RAS {location="Montpellier, France \\ Nizhny Novgorod, Russia"} - -I2S Doctorlal School -at the University of Montpellier -& L2C - -### Ph.D.~in Solid State Physics {date="Sep.~2014 \\ Dec.~2017"} - -Thesis: -[ - "Physical properties of HgCdTe-based heterostructures: - towards terahertz emission and detection" -](https://www.theses.fr/en/2017MONTS086) - -Implemented -a double-modulation technique, -enabling the extraction of critical magnetic fields -in a topological insulator. - -First to observe -[a temperature-driven phase transition](https://dx.doi.org/10.1103/PhysRevLett.120.086401) -in a HgTe/CdHgTe topological insulator -using magnetotransport. - -# Technical stack {.hidden} - ---- - -**Data analysis & presentation**: -[Python](https://www.python.org/), -[NumPy](https://numpy.org/), -[Pandas](https://pandas.pydata.org/), -[Xarray](https://xarray.dev/), -[SciPy](https://scipy.org/), -[Matplotlib](https://matplotlib.org/), -[hvPlot](https://hvplot.holoviz.org/), -[Plotly](https://plotly.com/python/), -[Bokeh](https://bokeh.org/), -[Panel](https://panel.holoviz.org/), -[Intake](https://intake.readthedocs.io) - -**Instrumentation integration & orchestration**: -[PyMeasure](https://pymeasure.readthedocs.io), -[Bluesky](https://blueskyproject.io/), -[yaq](https://yaq.fyi/), -[LabVIEW](https://www.ni.com/en/shop/labview.html) - -**Reporting**: -[Quarto](https://quarto.org/), -[Jupyter](https://jupyter.org/), -[Typst](https://typst.app/), -[LaTeX](https://www.latex-project.org/) - -**Programming**: -[VSCode](https://code.visualstudio.com/), -[Git](https://git-scm.com/), -[Linux](https://www.linux.com/what-is-linux/), -[Docker](https://www.docker.com/), -[PyTest](https://docs.pytest.org/), -[Pre-Commit](https://pre-commit.com/), -[GitLab CI/CD](https://gitlab.com/kadykov/), -[GitHub Actions](https://github.com/kadykov/), -[TDD](https://en.wikipedia.org/wiki/Test-driven_development), -[Devcontainers](https://containers.dev/) - -**Languages**: -English (upper-intermediate), -French ([upper-intermediate](https://www.duolingo.com/profile/aleksandrkadykov)), -Russian (native) - ---- - -# Selected publications {.hidden} - -1. Kadykov, A.M., Krishtopenko, S.S., Jouault, B. et al., - [*Temperature-Induced Topological Phase Transition in HgTe Quantum - Wells*](https://dx.doi.org/10.1103/PhysRevLett.120.086401), - **Physical Review Letters**, 120(8), *086401*, 2018 - -1. Kadykov, A.M., Torres, J., Krishtopenko, S.S. et al., [*Terahertz - imaging of Landau levels in HgTe-based topological - insulators*](https://dx.doi.org/10.1063/1.4955018), **Applied - Physics Letters**, 108(26), *262102*, 2016 - -1. Teppe, F., Marcinkiewicz, M., Krishtopenko, S.S. et al., - [*Temperature-driven massless Kane fermions in HgCdTe - crystals*](https://dx.doi.org/10.1038/ncomms12576), **Nature - Communications**, 7, *12576*, 2016 - -1. Kadykov, A.M., Teppe, F., Consejo, C. et al., [*Terahertz detection - of magnetic field-driven topological phase transition in HgTe-based - transistors*](https://dx.doi.org/10.1063/1.4932943), **Applied - Physics Letters**, 107(15), *152101*, 2015 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/kadykov-cv-fr.md b/kadykov-cv-fr.md deleted file mode 100644 index eda9097..0000000 --- a/kadykov-cv-fr.md +++ /dev/null @@ -1,464 +0,0 @@ ---- -author: Aleksandr KADYKOV -title: Ingénieur de recherche -email: cv@kadykov.com -github: kadykov -gitlab: kadykov -linkedin: aleksandr-kadykov -website: www.kadykov.com -date: "year: 1970, month: 1, day: 1" -keywords: - - résumé - - resume - - CV - - Curriculum vitae - - Research Software Engineer - - Research Engineer - - Ingénieur de Recherche - - Software Engineer - - THz - - terahertz - - THz-TDS - - spectroscopy - - cryogenics - - optics - - photonics - - measurements - - data analysis - - programming - - DevOps - - CI/CD - - TDD - - Test-Driven Development - - Python - - Jupyter - - NumPy - - Pandas - - Xarray - - Scipy - - Python array API - - scikit-learn - - PyTorch - - MATLAB - - Matplotlib - - hvPlot - - Plotly - - Bokeh - - Panel - - holoviz - - OriginPro - - PyMeasure - - Bluesky - - yaq - - LabVIEW - - Intake - - SQL - - Quarto - - Typst - - Pandoc - - LaTeX - - VSCode - - Git - - Linux - - Docker - - Docker-compose - - Zotero -links: - Multitel: https://www.multitel.be/expertises/photonique-appliquee/imagerie-spectroscopie-terahertz/ - ASBL: https://fr.wikipedia.org/wiki/Association_sans_but_lucratif - SAPHIRE: https://www.multitel.be/projets/saphire/ - LNE: https://www.lne.fr/fr/recherche-et-developpement/activites-r-et-d - EPIC: https://fr.wikipedia.org/wiki/%C3%89tablissement_public_%C3%A0_caract%C3%A8re_industriel_et_commercial_en_France - I2S: https://edi2s.umontpellier.fr/ - L2C: https://coulomb.umontpellier.fr/ - IPM: http://www.ipmras.ru/en/institute/scientific-departments/department-110/ - RAS: https://fr.wikipedia.org/wiki/Acad%C3%A9mie_des_sciences_de_Russie - TDS: https://fr.wikipedia.org/wiki/Spectroscopie_t%C3%A9rahertz_dans_le_domaine_temporel - THz: https://fr.wikipedia.org/wiki/T%C3%A9rahertz - térahertz: https://fr.wikipedia.org/wiki/T%C3%A9rahertz - hBN: https://fr.wikipedia.org/wiki/Nitrure_de_bore - HgCdTe: https://fr.wikipedia.org/wiki/Tellurure_de_mercure-cadmium - graphène: https://fr.wikipedia.org/wiki/Graph%C3%A8ne - hélium: https://fr.wikipedia.org/wiki/H%C3%A9lium_liquide - Python: https://github.com/search?q=language%3APython+author%3Akadykov&type=pullrequests - d'onde: https://doi.org/10.1063/1.4996966 - FTIR: https://fr.wikipedia.org/wiki/Spectroscopie_infrarouge_%C3%A0_transform%C3%A9e_de_Fourier - FAIR: https://fr.wikipedia.org/wiki/Fair_data - Mons: https://www.openstreetmap.org/#map=19/50.45756/3.92540 - Trappes: https://www.openstreetmap.org/#map=17/48.76090/1.98370 - Nizhnij: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Novgorod: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Thèse: https://www.theses.fr/en/2017MONTS086 ---- - -# Ingénieur de Recherche {photo='image("./photo.jpg", width: 120pt)'} - -Concevoir et exécuter des expériences, -analyser et présenter des données, -développer des logiciels scientifiques Python. - -Connaissances en -recherche appliquée et fondamentale -en photonique THz -et -magnéto-transport -dans les matériaux 2D. - -# Compétences clés {.hidden} - -- Analyse et présentation de données -- Conception et exécution d'expériences -- Intégration et orchestration d'instrumentation -- Développement de logiciels scientifiques Python - -# Expérience professionnelle {.hidden} - ---- - -## Multitel ASBL {location="Mons \\ Belgique"} - -_Centre d'innovation sans but lucratif -en photonique appliquée, IA, etc._ - -### Ingénieur de Recherche en Spectroscopie et Imagerie THz {date="Juil.~2021 \\ Août~2024"} - -Développé -un pipeline de données de spectroscopie THz en domaine temporel (THz-TDS) -avec un rapport signal-bruit amélioré, -en utilisant un filtrage avancé. - - -Développé -une méthode de traitement de données THz-TDS - -pour l'extraction de l'indice de réfraction et de l'épaisseur -dans les matériaux à faible absorption. - -Optimisé -la reconstruction de profils d'indice de réfraction -à partir de données THz-TDS -en déchargeant les calculs -sur une carte graphique (GPU) -et en utilisant des algorithmes avancés. - - - - -Automatisé -les flux de travail de laboratoire -en mettant en œuvre des outils Python -pour l'orchestration des mesures, -la gestion des données, -l'analyse, -et la présentation des résultats. - -Garanti les meilleures pratiques de développement logiciel -en mettant en œuvre des tests unitaires, -des pipelines CI/CD, -et de la documentation. - - - - - ---- - -## Laboratoire National de Métrologie et d'Essais {location="Trappes \\ France"} - -_Établissement public à caractère industriel et commercial (LNE)_ - -### Ingénieur de Recherche en Métrologie Électrique Quantique {date="Sept.~2018 \\ Sept.~2020"} - -Dirigé des mesures de magnéto-transport à faible bruit -et à basses températures -sur du graphène. - - - -Conçu -un package logiciel Python flexible, -optimisant l'orchestration de l'équipement scientifique. - -Participé -à la nano-fabrication -de piles de graphène/hBN. - - - ---- - -## Institute for Physics of Microstructures (IPM RAS) {location="Nijni Novgorod \\ Russie"} - -_Institut de recherche public -spécialisé en physique de l'état solide_ - -### Ingénieur de Recherche en Photonique de Hétérostructures à 2D à Faible Écart d'Énergie {date="Mai~2017 \\ Sept.~2018"} - -Dirigé -des mesures cryogéniques de photoluminescence et de photoconductivité -par spectroscopie infrarouge à transformée de Fourier (FTIR) -de puits quantiques HgTe/HgCdTe. - -A obtenu -une émission laser -dans des hétérostructures HgCdTe -à une longueur d'onde record. - ---- - -# Éducation {.hidden} - -## Laboratoire Charles Coulomb (L2C) & IPM RAS {location="Montpellier, France \\ Nijni Novgorod, Russie"} - -École doctorale I2S -de l'Université de Montpellier - -### Doctorat en Physique de l'État Solide {date="Sept.~2014 \\ Déc.~2017"} - -Thèse: -[ - Propriétés physiques d'hétérostructures à base de HgCdTe : - vers l'émission et la détection Terahertz -]("https://theses.fr/2017MONTS086/) - -A mis en œuvre -une technique de double modulation, -permettant l'extraction des champs magnétiques critiques -dans un isolant topologique. - -Premier à observer une -[transition de phase thermique](https://dx.doi.org/10.1103/PhysRevLett.120.086401) -dans un isolant topologique HgTe/CdHgTe -à l'aide de la magnéto-transport. - ---- - -# Compétences techniques {.hidden} - -**Analyse et présentation de données**: -[Python](https://www.python.org/), -[NumPy](https://numpy.org/), -[Pandas](https://pandas.pydata.org/), -[Xarray](https://xarray.dev/), -[SciPy](https://scipy.org/), -[Matplotlib](https://matplotlib.org/), -[hvPlot](https://hvplot.holoviz.org/), -[Plotly](https://plotly.com/python/), -[Bokeh](https://bokeh.org/), -[Panel](https://panel.holoviz.org/), -[Intake](https://intake.readthedocs.io) - -**Intégration et orchestration d'instrumentation**: -[PyMeasure](https://pymeasure.readthedocs.io), -[Bluesky](https://blueskyproject.io/), -[yaq](https://yaq.fyi/), -[LabVIEW](https://www.ni.com/en/shop/labview.html) - -**Élaboration de rapports**: -[Quarto](https://quarto.org/), -[Jupyter](https://jupyter.org/), -[Typst](https://typst.app/), -[LaTeX](https://www.latex-project.org/) - -**Développement de logiciels**: -[VSCode](https://code.visualstudio.com/), -[Git](https://git-scm.com/), -[Linux](https://www.linux.com/what-is-linux/), -[Docker](https://www.docker.com/), -[PyTest](https://docs.pytest.org/), - -[GitLab CI/CD](https://gitlab.com/kadykov/), -[GitHub Actions](https://github.com/kadykov/), -[TDD](https://en.wikipedia.org/wiki/Test-driven_development), -[Devcontainers](https://containers.dev/) - -**Langues**: -Anglais (avancé), -Français ([avancé](https://www.duolingo.com/profile/aleksandrkadykov)), -Russe - ---- - -# Sélection de publications {.hidden} - -1. Kadykov, A.M., Krishtopenko, S.S., Jouault, B. et al., - [*Temperature-Induced Topological Phase Transition in HgTe Quantum - Wells*](https://dx.doi.org/10.1103/PhysRevLett.120.086401), - **Physical Review Letters**, 120(8), *086401*, 2018 - -1. Kadykov, A.M., Torres, J., Krishtopenko, S.S. et al., [*Terahertz - imaging of Landau levels in HgTe-based topological - insulators*](https://dx.doi.org/10.1063/1.4955018), **Applied - Physics Letters**, 108(26), *262102*, 2016 - -1. Teppe, F., Marcinkiewicz, M., Krishtopenko, S.S. et al., - [*Temperature-driven massless Kane fermions in HgCdTe - crystals*](https://dx.doi.org/10.1038/ncomms12576), **Nature - Communications**, 7, *12576*, 2016 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/kadykov-letter-en.md b/kadykov-letter-en.md deleted file mode 100644 index 20daf6c..0000000 --- a/kadykov-letter-en.md +++ /dev/null @@ -1,223 +0,0 @@ ---- -author: Aleksandr KADYKOV -title: Letter to Future Employer -email: cv@kadykov.com -github: kadykov -gitlab: kadykov -linkedin: aleksandr-kadykov -website: www.kadykov.com -date: "year: 1970, month: 1, day: 1" -from: Mons, Belgium -# to: | -# Future Employer - -# Future Company - -# Future Country -keywords: - - CV - - Cover Letter - - résumé - - resume - - Curriculum vitae - - Research Software Engineer - - Research Engineer - - Software Engineer - - THz - - terahertz - - THz-TDS - - spectroscopy - - cryogenics - - optics - - photonics - - measurements - - data analysis - - programming - - DevOps - - CI/CD - - TDD - - Test-Driven Development - - Python - - Jupyter - - NumPy - - Pandas - - Xarray - - Scipy - - Python array API - - scikit-learn - - PyTorch - - MATLAB - - Matplotlib - - hvPlot - - Plotly - - Bokeh - - Panel - - holoviz - - OriginPro - - PyMeasure - - Bluesky - - yaq - - LabVIEW - - Intake - - SQL - - Quarto - - Typst - - Pandoc - - LaTeX - - VSCode - - Git - - Linux - - Docker - - Docker-compose - - Zotero -links: - Multitel: https://www.multitel.eu/expertise/applied-photonics/terahertz-spectroscopy-and-imaging/ - ASBL: https://fr.wikipedia.org/wiki/Association_sans_but_lucratif - SAPHIRE: https://www.multitel.eu/projects/saphire/ - LNE: https://www.lne.fr/en/research-and-development - EPIC: https://en.wikipedia.org/wiki/%C3%89tablissement_public_%C3%A0_caract%C3%A8re_industriel_et_commercial - I2S: https://edi2s.umontpellier.fr/ - L2C: https://coulomb.umontpellier.fr/?lang=en - IPM: http://www.ipmras.ru/en/institute/scientific-departments/department-110/ - RAS: https://en.wikipedia.org/wiki/Russian_Academy_of_Sciences - TDS: https://en.wikipedia.org/wiki/Terahertz_time-domain_spectroscopy - THz: https://en.wikipedia.org/wiki/Terahertz_radiation - hBN: https://en.wikipedia.org/wiki/Boron_nitride - HgCdTe: https://en.wikipedia.org/wiki/Mercury_cadmium_telluride - graphene: https://en.wikipedia.org/wiki/Graphene - helium: https://en.wikipedia.org/wiki/Liquid_helium - Python: https://github.com/search?q=language%3APython+author%3Akadykov&type=pullrequests - wavelength: https://doi.org/10.1063/1.4996966 - FTIR: https://en.wikipedia.org/wiki/Fourier-transform_infrared_spectroscopy - FAIR: https://en.wikipedia.org/wiki/FAIR_data - Mons: https://www.openstreetmap.org/#map=19/50.45756/3.92540 - Trappes: https://www.openstreetmap.org/#map=17/48.76090/1.98370 - Nizhny: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Novgorod: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Thesis: https://www.theses.fr/en/2017MONTS086 ---- - -# Letter to Future Employer - -Dear Future Employer, - -I'm writing -to offer you -a high-level view -of my career path, -highlighting my expertise -in photonics, -magnetotransport, -high-precision measurements, -and THz technology. - -## Photonics & Magnetotransport at L2C & IPM RAS - -I began my career in basic research -as a Research Engineer. - -My Ph.D. work, -a collaboration between -Laboratoire Charles Coulomb (L2C) -and the Institute for Physics of Microstructures (IPM RAS), -focused on photonics and magnetotransport -in HgTe/HgCdTe quantum wells. - -These heterostructures could serve dual purposes -depending on their configuration: -far-infrared emitters and detectors, -or topological insulators -with protected edge states. - -At IPM RAS, -I studied their potential -as far-infrared emitters and detectors, -using - -photoconductivity and photoluminescence. - -At L2C, -I explored their topological insulator states and transitions, -using THz detection and magnetotransport measurements. - -This work led -to the first observation of a topological phase transition -in these heterostructures -by magnetotransport, -as well as record-breaking laser emission wavelengths, -with results published in journals -like Nature Communications, PRL, PRB, and APL. - -## High-Precision Measurements at LNE - -After completing my Ph.D., -I moved to applied research -and worked for two years -at the Laboratoire national de métrologie et d'essais (LNE) -as a Research Engineer. - -Here, -I continued to explore magnetotransport properties of 2D systems, -and conducted low-noise, -high-precision -quantum Hall effect measurements in graphene. - -My contributions included -automating measurements -using a Python-based orchestration system, -nanofabrication of graphene/hBN stacks, -and optimizing cryogenic measurement systems -to reduce costs -by improving helium recuperation -and implementing a reliable dry helium-free cryogenic system. - -## THz Innovation at Multitel ASBL - -Next, -I spent over three years -as a Research Engineer at Multitel ASBL, -leading THz time-domain spectroscopy (THz-TDS) -and imaging activities. - -There, -I developed THz-based solutions -for industrial applications -such as non-destructive quality control -of humidity, thickness, or composition - -in industries such as pharmaceuticals, polymers, and biotechnology. - -As a result, -I developed new and improved existing methods -for extracting information from THz-TDS data, -including computationally cheap -preliminary estimation of thickness and refractive index -in low-absorption samples -and sensitivity curve-shaped -filtering with enhanced signal-to-noise ratio. - -I also implemented infrastructure -for reproducible research, -including Python tools for instrument integration, -FAIR -(findable, accessible, interoperable, and reusable) -data management, -and automated data analysis pipelines. - -Through this work, -I gained hands-on experience in software development, -including test-driven development, -CI/CD pipeline automation, -and Docker-based containerization. - -## What's Next? - -I'm now seeking new opportunities -to leverage my skills as a Research Engineer -to contribute to impactful projects. - -Could this be with your team? - -Thank you for your time and consideration. - -Best regards, diff --git a/kadykov-letter-fr.md b/kadykov-letter-fr.md deleted file mode 100644 index e162f59..0000000 --- a/kadykov-letter-fr.md +++ /dev/null @@ -1,227 +0,0 @@ ---- -author: Aleksandr KADYKOV -title: Lettre au Futur Employeur -email: cv@kadykov.com -github: kadykov -gitlab: kadykov -linkedin: aleksandr-kadykov -website: www.kadykov.com -date: "year: 1970, month: 1, day: 1" -from: Mons, Belgique -# to: | -# Future Employer - -# Future Company - -# Future Country -keywords: - - CV - - Cover Letter - - résumé - - resume - - Curriculum vitae - - Research Software Engineer - - Research Engineer - - Ingénieur de Recherche - - Software Engineer - - THz - - terahertz - - THz-TDS - - spectroscopy - - cryogenics - - optics - - photonics - - measurements - - data analysis - - programming - - DevOps - - CI/CD - - TDD - - Test-Driven Development - - Python - - Jupyter - - NumPy - - Pandas - - Xarray - - Scipy - - Python array API - - scikit-learn - - PyTorch - - MATLAB - - Matplotlib - - hvPlot - - Plotly - - Bokeh - - Panel - - holoviz - - OriginPro - - PyMeasure - - Bluesky - - yaq - - LabVIEW - - Intake - - SQL - - Quarto - - Typst - - Pandoc - - LaTeX - - VSCode - - Git - - Linux - - Docker - - Docker-compose - - Zotero -links: - Multitel: https://www.multitel.be/expertises/photonique-appliquee/imagerie-spectroscopie-terahertz/ - ASBL: https://fr.wikipedia.org/wiki/Association_sans_but_lucratif - SAPHIRE: https://www.multitel.be/projets/saphire/ - LNE: https://www.lne.fr/fr/recherche-et-developpement/activites-r-et-d - EPIC: https://fr.wikipedia.org/wiki/%C3%89tablissement_public_%C3%A0_caract%C3%A8re_industriel_et_commercial_en_France - I2S: https://edi2s.umontpellier.fr/ - L2C: https://coulomb.umontpellier.fr/ - IPM: http://www.ipmras.ru/en/institute/scientific-departments/department-110/ - RAS: https://fr.wikipedia.org/wiki/Acad%C3%A9mie_des_sciences_de_Russie - TDS: https://fr.wikipedia.org/wiki/Spectroscopie_t%C3%A9rahertz_dans_le_domaine_temporel - THz: https://fr.wikipedia.org/wiki/T%C3%A9rahertz - térahertz: https://fr.wikipedia.org/wiki/T%C3%A9rahertz - hBN: https://fr.wikipedia.org/wiki/Nitrure_de_bore - HgCdTe: https://fr.wikipedia.org/wiki/Tellurure_de_mercure-cadmium - graphène: https://fr.wikipedia.org/wiki/Graph%C3%A8ne - hélium: https://fr.wikipedia.org/wiki/H%C3%A9lium_liquide - Python: https://github.com/search?q=language%3APython+author%3Akadykov&type=pullrequests - d'onde: https://doi.org/10.1063/1.4996966 - FTIR: https://fr.wikipedia.org/wiki/Spectroscopie_infrarouge_%C3%A0_transform%C3%A9e_de_Fourier - FAIR: https://fr.wikipedia.org/wiki/Fair_data - Mons: https://www.openstreetmap.org/#map=19/50.45756/3.92540 - Trappes: https://www.openstreetmap.org/#map=17/48.76090/1.98370 - Nizhnij: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Novgorod: https://www.openstreetmap.org/#map=17/56.29878/43.97990 - Thèse: https://www.theses.fr/en/2017MONTS086 ---- - -# Lettre au Futur Employeur - -Cher Futur Employeur, - -Je vous écris -pour vous offrir une vue d'ensemble -de mon parcours professionnel, -en mettant en lumière mon expertise -en photonique, -magnetotransport, -mesures de haute précision, -et technologie THz. - -## Photonique et Magnetotransport au L2C et IPM~RAS - -J'ai commencé ma carrière dans la recherche fondamentale -en tant qu'Ingénieur de Recherche. - -Mon travail de thèse, -une collaboration entre -le Laboratoire Charles Coulomb (L2C) -et l'Institut de Physique des Microstructures (IPM~RAS), -portait sur la photonique et le magnetotransport -dans des puits quantiques HgTe/HgCdTe. - -Ces hétérostructures peuvent avoir une double utilité -selon leur configuration : -des émetteurs et détecteurs pour l'infrarouge lointain, -ou des isolants topologiques -avec des états de bord protégés. - -À l'IPM~RAS, -j'ai étudié leur potentiel -en tant qu'émetteurs et détecteurs pour l'infrarouge lointain, -en utilisant -la photoconductivité et la photoluminescence. - - -Au L2C, -j'ai exploré leurs états d'isolants topologiques et leurs transitions, -en utilisant la détection THz et des mesures de magnetotransport. - -Ce travail a conduit -à la première observation d'une transition de phase topologique -dans ces hétérostructures -par magnetotransport, -ainsi qu'à des longueurs d'émission laser record, -avec des résultats publiés dans des revues -telles que Nature Communications, PRL, PRB, et APL. - -## Mesures de Haute Précision au LNE - -Après avoir terminé ma thèse, -je suis passé à la recherche appliquée -et ai travaillé pendant deux ans -au Laboratoire national de métrologie et d'essais (LNE) -en tant qu'Ingénieur de Recherche. - -Là-bas, -j'ai continué à explorer les propriétés de magnetotransport des systèmes 2D, -et j'ai effectué des mesures de haute précision -et à faible bruit -de l'effet Hall quantique dans le graphène. - -Mes contributions comprenaient : - -- l'automatisation des mesures - à l'aide d'un système d'orchestration basé sur Python, -- la nanofabrication de structures graphène/hBN, -- et l'optimisation des systèmes de mesure cryogéniques - pour réduire les coûts, - en améliorant la récupération d'hélium - et en mettant en œuvre un système cryogénique fiable sans hélium liquide. - -## Innovation THz chez Multitel ASBL - -Ensuite, -j'ai passé plus de trois ans -en tant qu'Ingénieur de Recherche chez Multitel ASBL, -un centre d'innovation sans but lucratif, -où j'ai dirigé les activités térahertz -de spectroscopie dans le domaine temporel THz (THz-TDS) -et d'imagerie. - -Là-bas, -j'ai développé des solutions basées sur le THz -pour des applications industrielles -comme le contrôle qualité non destructif -de l'humidité, de l'épaisseur ou de la composition -dans des industries telles que la pharmacie, les polymères et la biotechnologie. - -À cette fin, -j'ai développé de nouvelles méthodes -et amélioré les méthodes existantes -pour extraire des informations à partir de données THz-TDS, -y compris : - -- une estimation préliminaire et rapide de l'épaisseur - et de l'indice de réfraction dans des échantillons faiblement absorbants, -- et un filtrage optimisé, basé sur la courbe de sensibilité, - avec un meilleur rapport signal/bruit. - -J'ai également mis en place une infrastructure -pour une recherche reproductible, -en développant des outils Python pour l'intégration d'instruments, -la gestion des données FAIR -(findable, accessible, interoperable, reusable), -et des pipelines d'analyse de données automatisés. - -Grâce à ce travail, -j'ai acquis une expérience pratique en développement logiciel, -y compris le développement piloté par les tests, -l'automatisation des pipelines CI/CD, -et la conteneurisation basée sur Docker. - -## Et ensuite ? - -Je suis maintenant à la recherche de nouvelles opportunités -pour mettre à profit mes compétences d'Ingénieur de Recherche -et contribuer à des projets impactants. - -Cela pourrait-il être avec votre équipe ? - - - -Cordialement, From 9436956f90d7f060d2521f7a2aee3b11e0daf7c9 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 10:35:04 +0000 Subject: [PATCH 17/63] Refactor devcontainer configuration: remove unnecessary user settings and add Bats testing framework --- .devcontainer/devcontainer.json | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2a697a5..e04e1ba 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,11 +5,7 @@ }, "features": { "ghcr.io/devcontainers/features/common-utils:2": { - "installZsh": "true", - "username": "vscode", - "userUid": "1000", - "userGid": "1000", - "upgradePackages": "true" + "username": "vscode" }, "ghcr.io/kadykov/devcontainer-features/pre-commit:1": {}, "ghcr.io/lukewiwa/features/shellcheck:0": {} @@ -27,7 +23,8 @@ "eamodio.gitlens", "sumneko.lua", "yinfei.luahelper", - "saoudrizwan.claude-dev" + "saoudrizwan.claude-dev", + "jetmartin.bats" ] } }, From f99e880e5b1476462976455b6dc36dc37e83045e Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 11:07:25 +0000 Subject: [PATCH 18/63] Refactor style references: update function calls to use the 'style' prefix for consistency across typst and lua files --- style.typ | 2 -- tests/filter/filters.bats | 10 +++++----- typst-cv.lua | 14 +++++++------- typst-cv.typ | 7 +++---- typst-letter.typ | 9 ++++----- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/style.typ b/style.typ index 1f15467..8a5f74e 100644 --- a/style.typ +++ b/style.typ @@ -1,7 +1,5 @@ #import "@preview/fontawesome:0.5.0": * -#let author = "Aleksandr KADYKOV" - #let text-color = black #let background-color = luma(100%, 0%) // Transparent white #let primary-color = rgb("#3A468C") diff --git a/tests/filter/filters.bats b/tests/filter/filters.bats index 80b3bc2..05c18f9 100755 --- a/tests/filter/filters.bats +++ b/tests/filter/filters.bats @@ -60,7 +60,7 @@ setup() { local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" run eval "$cmd" assert_success - assert_output --partial 'side: profile-photo(image("photo.png", width: 100pt))' + assert_output --partial 'side: style.profile-photo(image("photo.png", width: 100pt))' } @test "filter/typst-cv: handles hidden class" { @@ -68,7 +68,7 @@ setup() { local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" run eval "$cmd" assert_success - assert_output --partial '#hidden-heading()' + assert_output --partial '#style.hidden-heading()' } @test "filter/typst-cv: handles location attribute" { @@ -77,7 +77,7 @@ setup() { run eval "$cmd" assert_success # Check for the specific function call and the content (actual output has single backslash) - assert_output --partial 'side: event-date()[City \ Country]' + assert_output --partial 'side: style.event-date()[City \ Country]' } @test "filter/typst-cv: handles date attribute" { @@ -85,7 +85,7 @@ setup() { local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" run eval "$cmd" assert_success - assert_output --partial 'side: company-location()[2023 - Present]' + assert_output --partial 'side: style.company-location()[2023 - Present]' } @test "filter/typst-cv: handles ordered list" { @@ -102,7 +102,7 @@ setup() { run eval "$cmd" assert_success # Check date was processed (include closing parenthesis) - assert_output --partial 'side: company-location()[Date First])' + assert_output --partial 'side: style.company-location()[Date First])' # Check location was NOT processed on the same header refute_output --partial 'Header With Both.*side: event-date()' # Keep .* here for refute flexibility } diff --git a/typst-cv.lua b/typst-cv.lua index aac5020..501ab05 100644 --- a/typst-cv.lua +++ b/typst-cv.lua @@ -8,21 +8,21 @@ function Header(el) for key, value in pairs(el.attributes) do if key == "date" then - table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.body-side([")) table.insert(typst_blocks, el) table.insert(typst_blocks, - pandoc.RawBlock("typst", string.format("], side: company-location()[%s])", value))) + pandoc.RawBlock("typst", string.format("], side: style.company-location()[%s])", value))) return typst_blocks elseif key == "location" then - table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.body-side([")) table.insert(typst_blocks, el) - table.insert(typst_blocks, pandoc.RawBlock("typst", string.format("], side: event-date()[%s])", value))) + table.insert(typst_blocks, pandoc.RawBlock("typst", string.format("], side: style.event-date()[%s])", value))) return typst_blocks elseif key == "photo" then - table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.body-side([")) table.insert(typst_blocks, el) table.insert(typst_blocks, - pandoc.RawBlock("typst", string.format("], side: profile-photo(%s))", value))) + pandoc.RawBlock("typst", string.format("], side: style.profile-photo(%s))", value))) return typst_blocks end end @@ -30,7 +30,7 @@ function Header(el) if el.classes:includes("hidden") then local typst_blocks = {} - table.insert(typst_blocks, pandoc.RawBlock("typst", "#hidden-heading()[")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.hidden-heading()[")) table.insert(typst_blocks, el) table.insert(typst_blocks, pandoc.RawBlock("typst", "]")) return typst_blocks diff --git a/typst-cv.typ b/typst-cv.typ index dac50a0..37b0237 100644 --- a/typst-cv.typ +++ b/typst-cv.typ @@ -1,11 +1,10 @@ -// #import "style.typ": * -#import "@local/pandoc-cv:0.1.0": * +#import "style.typ" // Import the style module // Construct datetime object from structured date if provided via YAML map, else use today #let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ // Call the function from `style.typ` and pass variables to set up the document style -#show: setup-style.with( +#show: style.setup-style.with( $if(title)$title: "$title$", $endif$ // Comma after if present $if(author)$author: "$author$", $endif$ // Comma after if present $if(email)$email: "$email$".replace("\\", ""), $endif$ // Comma after if present @@ -18,7 +17,7 @@ hyphenate: auto // Last argument, no trailing comma ) -#block(width: bodywidth)[ +#block(width: style.bodywidth)[ $body$ diff --git a/typst-letter.typ b/typst-letter.typ index eb397d1..1abd0ab 100644 --- a/typst-letter.typ +++ b/typst-letter.typ @@ -1,4 +1,4 @@ -#import "@local/pandoc-cv:0.1.0": * +#import "style.typ" // Import the style module // Construct datetime object from structured date if provided via YAML map, else use today #let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ @@ -17,7 +17,7 @@ } // Call the function from `style.typ` and pass variables to set up the document style -#show: setup-style.with( +#show: style.setup-style.with( $if(title)$title: "$title$", $endif$ // Comma after if present $if(author)$author: "$author$", $endif$ // Comma after if present $if(email)$email: "$email$".replace("\\", ""), $endif$ // Comma after if present @@ -49,16 +49,15 @@ box(height: intro-height)[#from-content] h(1fr) - box(height: intro-height, width: bodywidth)[#to-content] + box(height: intro-height, width: style.bodywidth)[#to-content] } #v(1em) #align(right)[ - #block(width: bodywidth)[ + #block(width: style.bodywidth)[ #set align(left) $body$ - #author ] ] From 5331e5c3822d1a24afb6d71942154e76927b3ed4 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 11:56:56 +0000 Subject: [PATCH 19/63] Enhance Dockerfile and tests: add poppler-utils for PDF content checks, refactor filter tests, and implement new CV fixture to validate absence of optional footer links. --- Dockerfile | 1 + memory-bank/activeContext.md | 12 ++-- style.typ | 47 ++++++++++----- tests/fixtures/example-cv-no-optional.md | 67 +++++++++++++++++++++ tests/test_e2e.sh | 77 ++++++++++++++++++++++++ 5 files changed, 183 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/example-cv-no-optional.md diff --git a/Dockerfile b/Dockerfile index 89b340c..8b9e5c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN dnf update -yq \ ibm-plex-serif-fonts-${IBM_PLEX_VERSION} \ fontawesome-6-brands-fonts-${FONTAWESOME_VERSION} \ fontawesome-6-free-fonts-${FONTAWESOME_VERSION} \ + poppler-utils \ # Added for pdftotext used in E2E tests && dnf clean all ENV PANDOC_DATA_DIR=/usr/share/pandoc-${PANDOC_VERSION}/ diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index d609872..0128cf0 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -16,10 +16,10 @@ Phase 1, focusing on adding tests, depersonalizing examples, and updating CI, is - Removed old personalized files (`kadykov-*.md`, `photo.jpg`). - **Testing Implementation:** - Added unit tests for `build.sh` using Bats (`tests/unit/build_sh.bats`), debugging path and execution issues. - - Added filter tests (`tests/test_filters.sh`) comparing Pandoc+filter output against snapshots (using templates where necessary). Updated snapshots to match actual Pandoc output formatting. - - Added E2E smoke tests (`tests/test_e2e.sh`) verifying successful PDF generation for example files. Debugged issues related to YAML date format and image paths in templates/fixtures. + - Refactored filter tests into Bats (`tests/filter/filters.bats`) using `grep`/`assert_output --partial` for robustness instead of fragile snapshots. + - Added E2E smoke tests (`tests/test_e2e.sh`) verifying successful PDF generation. Enhanced E2E tests using `pdftotext` (via `poppler-utils` added to Dockerfile) to check for presence of hidden headings and absence of optional footer links. Debugged various issues related to YAML format, paths, template logic, and test implementation. - **Documentation:** Updated `README.md` to clarify heading structure, linkification, metadata override workflow, and point to new example files. -- **CI Update:** Modified `.github/workflows/ci.yml` to run the new test suites (unit, filter, e2e) inside the Docker container and removed the old GitHub Pages deployment job for CVs. Ensured example PDFs are built and uploaded for release artifacts. +- **CI Update:** Modified `.github/workflows/ci.yml` to run the new test suites (unit, filter, e2e) inside the Docker container (which now includes `poppler-utils`) and removed the old GitHub Pages deployment job for CVs. Ensured example PDFs are built and uploaded for release artifacts. - **Development Tooling:** Added `justfile` for convenient local execution of linters and test suites. - **Memory Bank:** Updated `activeContext.md`, `progress.md`, `techContext.md`. @@ -28,8 +28,10 @@ Phase 1, focusing on adding tests, depersonalizing examples, and updating CI, is - Decided *not* to implement `--output-format typst` in `build.sh` at this time due to difficulties in reliably generating/testing the `.typ` output via the script. Integration testing relies on filter tests and E2E PDF tests. - Noted suggestions for future filter (`typst-cv.lua`) refactoring (fixing function names, adding generic side content, multiple attributes, div support) but postponed implementation until after establishing baseline tests. - Corrected understanding of YAML date format and relative paths for Typst compilation. +- **Bug Fix:** Corrected default values (`"default"` -> `none`) and added conditional logic in `style.typ` footer to prevent displaying icons/links for optional fields (github, gitlab, linkedin, website) if they are not provided in YAML or via overrides. +- **Template Imports:** Confirmed templates use direct `#import "style.typ"` as updated by user. ## Immediate Next Steps -- Update `progress.md` and `techContext.md`. -- Present the completed Phase 1 work to the user. +- Update `progress.md`. +- Present the completed Phase 1 work (including bug fix) to the user. diff --git a/style.typ b/style.typ index 8a5f74e..fc571f2 100644 --- a/style.typ +++ b/style.typ @@ -22,12 +22,12 @@ #let setup-style( // These values come from the Pandoc template ($variable$) and act as fallbacks author: "Default Author", - email: "default@example.com", - title: "Default Title", - website: "example.com", - github: "default", - gitlab: "default", - linkedin: "default", + email: "default@example.com", // Keep default, likely always needed + title: "Default Title", // Keep default + website: none, // Change default to none + github: none, // Change default to none + gitlab: none, // Change default to none + linkedin: none, // Change default to none keywords: ("Default",), language: "en", date: auto, @@ -99,16 +99,31 @@ #v(-1em) #line(length: 100%, stroke: 0.5pt) #v(-0.5em) - // Use effective values for footer links - #text(fill: primary-color)[#fa-arrow-up-right-from-square()] - #link("https://" + effective-website)[#effective-website] - #h(1fr) - #text(fill: black)[#fa-github()] - #link("https://github.com/" + effective-github)[#effective-github] | - #text(fill: orange)[#fa-gitlab()] - #link("https://gitlab.com/" + effective-gitlab)[#effective-gitlab] | - #text(fill: blue)[#fa-linkedin()] - #link("https://www.linkedin.com/in/" + effective-linkedin)[#effective-linkedin] + // Use effective values for footer links, checking for none + #if effective-website != none { + text(fill: primary-color)[#fa-arrow-up-right-from-square()] + link("https://" + effective-website)[#effective-website] + } + #h(1fr) // Keep horizontal space regardless + // Group social links and add separators conditionally + #let social-links = () + #if effective-github != none { + social-links.push([#text(fill: black)[#fa-github()] #link( + "https://github.com/" + effective-github, + )[#effective-github]]) + } + #if effective-gitlab != none { + social-links.push([#text(fill: orange)[#fa-gitlab()] #link( + "https://gitlab.com/" + effective-gitlab, + )[#effective-gitlab]]) + } + #if effective-linkedin != none { + social-links.push([#text(fill: blue)[#fa-linkedin()] #link( + "https://www.linkedin.com/in/" + effective-linkedin, + )[#effective-linkedin]]) + } + // Display social links with separators + #social-links.join([ | ]) ], ) diff --git a/tests/fixtures/example-cv-no-optional.md b/tests/fixtures/example-cv-no-optional.md new file mode 100644 index 0000000..a3e3154 --- /dev/null +++ b/tests/fixtures/example-cv-no-optional.md @@ -0,0 +1,67 @@ +--- +title: Curriculum Vitae - No Optional Footer Links +author: Jane Doe Minimal +date: # Use structured date + year: 2025 + month: 4 + day: 26 +email: jane.doe.minimal@example.com +# No website, github, gitlab, linkedin here +# Optional: Define links used in the document +links: + Typst: https://typst.app/ + Pandoc: https://pandoc.org/ +--- + +# Jane Doe Minimal {photo='image("tests/fixtures/placeholder-photo.png", width: 120pt)'} + +## Summary + +This CV omits optional footer links (website, github, gitlab, linkedin) to test conditional display logic. + +## Experience {.hidden} + +### Example Corp {location="Remote"} +*Leading provider of innovative examples* + +#### Senior Example Engineer {date="2022 - Present"} +- Developed and maintained key features. + +### Another Company Inc. {location="Example City"} +*Pioneers in fictional services* + +#### Example Intern {date="Summer 2021"} +- Assisted the development team. + +## Education {.hidden} + +### University of Example {location="Example City, EX"} + +#### M.S. in Computer Science {date="2020 - 2022"} +*Thesis: Advanced Topics in Placeholders* + +### Example College {location="Another Town, AT"} + +#### B.S. in Software Engineering {date="2016 - 2020"} +*Minor: Fictional Studies* + +## Skills + +- **Programming:** Python, JavaScript, Typst, Shell Scripting +- **Tools:** Git, Docker, Pandoc +- **Concepts:** Agile Development, Test-Driven Development + +## Projects + +### Typst CV Generator + +A project to generate CVs and letters using Typst. + +### Placeholder Project + +Another example project demonstrating skills. + +## Languages + +- English (Native) +- French (Conversational) diff --git a/tests/test_e2e.sh b/tests/test_e2e.sh index 9677ffe..c950e53 100755 --- a/tests/test_e2e.sh +++ b/tests/test_e2e.sh @@ -42,6 +42,31 @@ if [ ! -s "$CV_EXPECTED_PDF" ]; then fi echo "example-cv.md PDF generated successfully." +# --- Check example-cv.md PDF Content (Hidden Headings) --- +echo "Checking example-cv.md PDF text content..." +CV_TXT_OUTPUT=$(mktemp) +if command -v pdftotext >/dev/null 2>&1; then + pdftotext "$CV_EXPECTED_PDF" "$CV_TXT_OUTPUT" + # Check that hidden headings ARE present in the text layer + if ! grep -q "Experience" "$CV_TXT_OUTPUT"; then + echo "Error: Hidden heading 'Experience' not found in PDF text output." >&2 + cat "$CV_TXT_OUTPUT" >&2 + rm "$CV_TXT_OUTPUT" + exit 1 + fi + if ! grep -q "Education" "$CV_TXT_OUTPUT"; then + echo "Error: Hidden heading 'Education' not found in PDF text output." >&2 + cat "$CV_TXT_OUTPUT" >&2 + rm "$CV_TXT_OUTPUT" + exit 1 + fi + rm "$CV_TXT_OUTPUT" + echo "Hidden heading checks passed." +else + echo "Warning: pdftotext not found. Skipping hidden heading content check." >&2 +fi + + # --- Test example-letter.md PDF Generation --- echo "Testing example-letter.md -> PDF..." LETTER_FIXTURE="${FIXTURES_DIR}/example-letter.md" @@ -69,5 +94,57 @@ fi echo "example-letter.md PDF generated successfully." +# --- Test example-cv-no-optional.md PDF Generation (Check Footer) --- +echo "Testing example-cv-no-optional.md -> PDF (No Optional Footer Links)..." +NOOPT_FIXTURE="${FIXTURES_DIR}/example-cv-no-optional.md" +NOOPT_EXPECTED_FILENAME="example-cv-no-optional.pdf" +NOOPT_EXPECTED_PDF="${TEST_OUTPUT_DIR}/${NOOPT_EXPECTED_FILENAME}" +NOOPT_TXT_OUTPUT=$(mktemp) + +# Run build.sh to generate PDF output +"$BUILD_SCRIPT" --output-dir "$TEST_OUTPUT_DIR" --output "$NOOPT_EXPECTED_FILENAME" "$NOOPT_FIXTURE" +exit_code=$? + +if [ $exit_code -ne 0 ]; then + echo "Error: build.sh failed for example-cv-no-optional.md PDF generation" >&2 + exit 1 +fi + +# Check if PDF exists and has size > 0 +if [ ! -f "$NOOPT_EXPECTED_PDF" ]; then + echo "Error: Expected PDF file not found: $NOOPT_EXPECTED_PDF" >&2 + exit 1 +fi +if [ ! -s "$NOOPT_EXPECTED_PDF" ]; then + echo "Error: Generated PDF file is empty: $NOOPT_EXPECTED_PDF" >&2 + exit 1 +fi + +# Convert PDF to text and check for absence of optional links +# Requires poppler-utils (pdftotext) to be installed +if command -v pdftotext >/dev/null 2>&1; then + pdftotext "$NOOPT_EXPECTED_PDF" "$NOOPT_TXT_OUTPUT" + # Check that specific URL parts or usernames are NOT present + if grep -q -e "github.com/" -e "gitlab.com/" -e "linkedin.com/in/" "$NOOPT_TXT_OUTPUT"; then + echo "Error: Optional footer links (GitHub/GitLab/LinkedIn) were found in PDF text output when they should be absent." >&2 + cat "$NOOPT_TXT_OUTPUT" >&2 + rm "$NOOPT_TXT_OUTPUT" + exit 1 + fi + # Check for website (might be harder if URL is generic like example.com) + # Let's assume the website field itself shouldn't appear if omitted + if grep -q -i "example.com" "$NOOPT_TXT_OUTPUT"; then + # This check might be too broad, but let's try it. The fixture uses example.com in links: section too. + # A better check might be needed if this causes false positives. + # echo "Warning: Found 'example.com', potentially the website link?" >&2 + : # Placeholder, maybe refine later + fi + rm "$NOOPT_TXT_OUTPUT" + echo "example-cv-no-optional.md PDF checks passed." +else + echo "Warning: pdftotext not found. Skipping footer content check for example-cv-no-optional.md." >&2 +fi + + echo "--- E2E Smoke Tests Passed ---" exit 0 From ac43918808fc8e8877bd542784ac496eea195f29 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 12:06:25 +0000 Subject: [PATCH 20/63] Add poppler-utils to Dockerfile for PDF processing in E2E tests --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8b9e5c4..c6f1811 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN dnf update -yq \ ibm-plex-serif-fonts-${IBM_PLEX_VERSION} \ fontawesome-6-brands-fonts-${FONTAWESOME_VERSION} \ fontawesome-6-free-fonts-${FONTAWESOME_VERSION} \ - poppler-utils \ # Added for pdftotext used in E2E tests + poppler-utils \ && dnf clean all ENV PANDOC_DATA_DIR=/usr/share/pandoc-${PANDOC_VERSION}/ From 0275a4406d835db434df18e120cbd8e8e4936e24 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 12:36:09 +0000 Subject: [PATCH 21/63] Update pre-commit hooks and enhance shellcheck directives in build and test scripts --- .pre-commit-config.yaml | 9 +++++++-- build.sh | 4 ++++ tests/test_e2e.sh | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 179a789..dc4c45b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,11 +25,16 @@ repos: # requires `shellcheck` to work properly # https://github.com/rhysd/actionlint/issues/477 - repo: https://github.com/rhysd/actionlint - rev: v1.7.4 + rev: v1.7.7 hooks: - id: actionlint - repo: https://github.com/google/yamlfmt - rev: v0.14.0 + rev: v0.16.0 hooks: - id: yamlfmt + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck diff --git a/build.sh b/build.sh index f27a149..3c1050e 100755 --- a/build.sh +++ b/build.sh @@ -58,6 +58,7 @@ while [ $# -gt 0 ]; do key=$(echo "$2" | cut -d= -f1 | tr '[:upper:]' '[:lower:]') value=$(echo "$2" | cut -d= -f2-) if [ -z "$key" ] || [ -z "$value" ]; then echo "Error: Invalid --set format. Use KEY=VALUE." >&2; usage; fi + # shellcheck disable=SC2089 typst_input_args="$typst_input_args --input $key=\"$value\"" use_override_pipeline=true shift 2 @@ -150,6 +151,7 @@ fi # --- Build Commands --- # Check if PANDOC_DATA_DIR is set, provide default if not (useful if run outside Docker) PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-/usr/share/pandoc}" +# shellcheck disable=SC2089 pandoc_base="pandoc --data-dir=\"$PANDOC_DATA_DIR\" --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" # --- Execute Pipeline --- @@ -160,6 +162,7 @@ if [ "$use_override_pipeline" = true ]; then # Construct the pandoc part of the pipe pandoc_cmd_part="$pandoc_base --to=typst --template=$template_file $input_arg" # Execute pandoc part directly and pipe to typst compile + # shellcheck disable=SC2086,SC2090 # We want word splitting; quotes in var are for target cmd $pandoc_cmd_part | typst compile $typst_input_args - $output_arg else echo "Info: Using direct pipeline (Pandoc -> PDF)." >&2 @@ -168,6 +171,7 @@ else pandoc_cmd_part2="$input_arg" # $output_arg contains '-o path' or is empty if default CWD output # Execute directly, let shell handle splitting of $output_arg + # shellcheck disable=SC2086,SC2090 # We want word splitting; quotes in var are for target cmd $pandoc_cmd_part1 "$pandoc_cmd_part2" $output_arg fi diff --git a/tests/test_e2e.sh b/tests/test_e2e.sh index c950e53..f226810 100755 --- a/tests/test_e2e.sh +++ b/tests/test_e2e.sh @@ -10,6 +10,7 @@ TEST_OUTPUT_DIR=$(mktemp -d) echo "--- Running E2E Smoke Tests (PDF Output) ---" +# shellcheck disable=SC2317 # Function appears unreachable but is called by trap cleanup() { echo "Cleaning up temporary directory: $TEST_OUTPUT_DIR" rm -rf "$TEST_OUTPUT_DIR" From 23232687a0fdf7b6ff07260860be683956e9d3e5 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 15:00:17 +0000 Subject: [PATCH 22/63] Remove unnecessary script execution step in CI workflow for cleaner test runs --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce293ac..f83149d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,10 +69,6 @@ jobs: - name: Run tests inside the container run: | echo "Running tests..." - # Make scripts executable - docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - sh -c "chmod +x ${{ env.TEST_DIR }}/*.sh" - # Execute test suites docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ bats ${{ env.TEST_DIR }}/unit/build_sh.bats From 15897ed9f768d8ee921c6cf9e909a69d6b898ce0 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sat, 26 Apr 2025 15:29:05 +0000 Subject: [PATCH 23/63] Add code checkout step before running tests in CI workflow --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f83149d..d8775bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,9 @@ jobs: cache-from: type=registry,ref=${{ env.IMAGE }}:testing # Cache from previous testing tag if exists cache-to: type=inline + - name: Checkout code + uses: actions/checkout@v4 + - name: Run tests inside the container run: | echo "Running tests..." From 894a2f074430ed1ffe2a969948fa5918f4aa7290 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sun, 27 Apr 2025 15:23:15 +0000 Subject: [PATCH 24/63] Add section separators in example CV for improved readability --- tests/fixtures/example-cv.md | 6 ++++++ typst-cv.typ | 1 + 2 files changed, 7 insertions(+) diff --git a/tests/fixtures/example-cv.md b/tests/fixtures/example-cv.md index 5f9f4fa..f5d95b7 100644 --- a/tests/fixtures/example-cv.md +++ b/tests/fixtures/example-cv.md @@ -25,6 +25,8 @@ private_info: This is private info from YAML A highly motivated and skilled professional seeking a challenging position. Experienced in various technologies and methodologies. Eager to contribute to innovative projects. Check my website. +--- + ## Experience {.hidden} ### Example Corp {location="Remote"} @@ -42,6 +44,8 @@ A highly motivated and skilled professional seeking a challenging position. Expe - Assisted the development team with various tasks including testing and documentation. - Gained practical experience in the software development lifecycle and agile methodologies. +--- + ## Education {.hidden} ### University of Example {location="Example City, EX"} @@ -56,6 +60,8 @@ Relevant coursework: Advanced Algorithms, Distributed Systems, Machine Learning. *Minor: Fictional Studies* Graduated with honors. Capstone project involved developing a sample application. +--- + ## Skills - **Programming:** Python, JavaScript, Typst, Shell Scripting diff --git a/typst-cv.typ b/typst-cv.typ index 37b0237..7cfec23 100644 --- a/typst-cv.typ +++ b/typst-cv.typ @@ -1,4 +1,5 @@ #import "style.typ" // Import the style module +#let horizontalrule = style.horizontalrule // Use the horizontal rule from the style module // Construct datetime object from structured date if provided via YAML map, else use today #let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ From 3f45db2986f02b8bebcb9fd10b85987dc4cc7e3c Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Sun, 27 Apr 2025 15:26:02 +0000 Subject: [PATCH 25/63] Add Dockerfile for font installation and update devcontainer configuration - Create Dockerfile to install fonts using fontist - Update devcontainer.json to reference the new Dockerfile and adjust context - Add devcontainer.env to .gitignore - Introduce fontist-manifest.yml for font management - Modify style.typ to set font weight for strong text --- .devcontainer/Dockerfile | 46 +++++++++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 18 +++++++------ .gitignore | 1 + fontist-manifest.yml | 8 ++++++ style.typ | 5 +++- 5 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 fontist-manifest.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..e3d8340 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,46 @@ +# === BUILDER STAGE: Install fonts with fontist === +FROM ruby:slim AS fonts-builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Install fontist and required fonts +RUN gem install fontist --no-document + +COPY fontist-manifest.yml . + +RUN fontist update --quiet \ + && fontist manifest-install --quiet fontist-manifest.yml \ + && fontist cache clear --quiet \ + && chmod -R 644 /root/.fontist/fonts + +# === FINAL STAGE: Alpine with copied fonts === +FROM alpine:3.21 + +RUN apk add --no-cache \ + bash \ + git \ + pandoc \ + typst \ + shellcheck \ + just \ + python3 \ + uv \ + pre-commit \ + nodejs \ + npm \ + fontconfig \ + font-awesome \ + font-awesome-free \ + font-awesome-brands \ + && rm -rf /var/cache/apk/* + +# Copy all fonts from the builder stage at once +COPY --from=fonts-builder /root/.fontist/fonts/ /usr/share/fonts/fontist/ + +ENV TYPST_FONT_PATHS=/usr/share/fonts + +# Update font cache +RUN fc-cache -fv diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e04e1ba..50c0abf 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,16 +1,20 @@ { - "name": "typst", + "name": "${localWorkspaceFolderBasename}-devcontainer", "build": { - "dockerfile": "../Dockerfile" + "dockerfile": "./Dockerfile", + "context": ".." }, + "runArgs": [ + "--env-file", + ".devcontainer/devcontainer.env" + ], "features": { "ghcr.io/devcontainers/features/common-utils:2": { "username": "vscode" }, - "ghcr.io/kadykov/devcontainer-features/pre-commit:1": {}, - "ghcr.io/lukewiwa/features/shellcheck:0": {} + "ghcr.io/cirolosapio/devcontainers-features/alpine-docker-outside-of-docker:0": {} }, - "postCreateCommand": "pre-commit install", + "postCreateCommand": "sudo chmod 666 /var/run/docker.sock && pre-commit install", "remoteUser": "vscode", "customizations": { "vscode": { @@ -18,9 +22,7 @@ "ms-azuretools.vscode-docker", "myriad-dreamin.tinymist", "skellock.just", - "ritwickdey.LiveServer", "GitHub.vscode-github-actions", - "eamodio.gitlens", "sumneko.lua", "yinfei.luahelper", "saoudrizwan.claude-dev", @@ -30,7 +32,7 @@ }, "mounts": [ { - "source": "vscode-home", + "source": "${localWorkspaceFolderBasename}-data", "target": "/home/vscode", "type": "volume" } diff --git a/.gitignore b/.gitignore index 8b19f98..824b37c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pdf .env temp.* +devcontainer.env diff --git a/fontist-manifest.yml b/fontist-manifest.yml new file mode 100644 index 0000000..5db8c25 --- /dev/null +++ b/fontist-manifest.yml @@ -0,0 +1,8 @@ +--- +IBM Plex Serif: + - Regular + - Italic +IBM Plex Serif SemiBold: + - Regular +Fira Sans Medium: + - Regular diff --git a/style.typ b/style.typ index fc571f2..91847db 100644 --- a/style.typ +++ b/style.typ @@ -63,7 +63,10 @@ hyphenate: auto, ) - show strong: set text(font: "IBM Plex Serif SmBld") + show strong: set text( + font: "IBM Plex Serif", + weight: "semibold", + ) set document( title: effective-title, From 30f7b3e45dca9513849364da05d8176beb096f2b Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Mon, 28 Apr 2025 22:24:13 +0000 Subject: [PATCH 26/63] Enhance Dockerfile and Typst templates for improved development setup - Added poppler-utils to Dockerfile for additional functionality. - Set up local Typst package symlink for development in Dockerfile. - Updated Typst templates to use local package functions and improved comments for clarity. --- .devcontainer/Dockerfile | 13 +++++++++++++ typst-cv.lua | 14 +++++++------- typst-cv.typ | 40 +++++++++++++++++++++++++--------------- typst-letter.typ | 18 ++++++++++++++---- 4 files changed, 59 insertions(+), 26 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e3d8340..d131c49 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -26,6 +26,7 @@ RUN apk add --no-cache \ typst \ shellcheck \ just \ + poppler-utils \ python3 \ uv \ pre-commit \ @@ -44,3 +45,15 @@ ENV TYPST_FONT_PATHS=/usr/share/fonts # Update font cache RUN fc-cache -fv + +# --- Setup Local Typst Package Symlink for Development --- +# Define the target package path using the standard ENV var name +ENV TYPST_PACKAGE_PATH=/usr/local/share/typst/packages +ENV DEV_PKG_DIR=$TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0 +# Create the directory structure +RUN mkdir -p $DEV_PKG_DIR +# Create symlinks from the workspace files to the package directory +# This assumes the devcontainer mounts the workspace at /workspaces/typstCV +# Use ln -sf to force link creation/overwrite if it exists +RUN ln -sf /workspaces/typstCV/style.typ $DEV_PKG_DIR/style.typ \ + && ln -sf /workspaces/typstCV/typst.toml $DEV_PKG_DIR/typst.toml diff --git a/typst-cv.lua b/typst-cv.lua index 501ab05..aac5020 100644 --- a/typst-cv.lua +++ b/typst-cv.lua @@ -8,21 +8,21 @@ function Header(el) for key, value in pairs(el.attributes) do if key == "date" then - table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.body-side([")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) table.insert(typst_blocks, el) table.insert(typst_blocks, - pandoc.RawBlock("typst", string.format("], side: style.company-location()[%s])", value))) + pandoc.RawBlock("typst", string.format("], side: company-location()[%s])", value))) return typst_blocks elseif key == "location" then - table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.body-side([")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) table.insert(typst_blocks, el) - table.insert(typst_blocks, pandoc.RawBlock("typst", string.format("], side: style.event-date()[%s])", value))) + table.insert(typst_blocks, pandoc.RawBlock("typst", string.format("], side: event-date()[%s])", value))) return typst_blocks elseif key == "photo" then - table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.body-side([")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) table.insert(typst_blocks, el) table.insert(typst_blocks, - pandoc.RawBlock("typst", string.format("], side: style.profile-photo(%s))", value))) + pandoc.RawBlock("typst", string.format("], side: profile-photo(%s))", value))) return typst_blocks end end @@ -30,7 +30,7 @@ function Header(el) if el.classes:includes("hidden") then local typst_blocks = {} - table.insert(typst_blocks, pandoc.RawBlock("typst", "#style.hidden-heading()[")) + table.insert(typst_blocks, pandoc.RawBlock("typst", "#hidden-heading()[")) table.insert(typst_blocks, el) table.insert(typst_blocks, pandoc.RawBlock("typst", "]")) return typst_blocks diff --git a/typst-cv.typ b/typst-cv.typ index 7cfec23..cb3c6ed 100644 --- a/typst-cv.typ +++ b/typst-cv.typ @@ -1,24 +1,34 @@ -#import "style.typ" // Import the style module -#let horizontalrule = style.horizontalrule // Use the horizontal rule from the style module +$-- ========================================================================== -- +$-- WARNING: PANDOC TEMPLATE - NOT PURE TYPST -- +$-- ========================================================================== -- +$-- This file uses Pandoc's templating syntax (e.g., $variable$, $if(...)$). -- +$-- It will NOT pass Typst linting or compilation directly. -- +$-- Pandoc processes this template *before* passing the result to Typst. -- +$-- Ignore Typst linter errors related to Pandoc syntax in this file. -- +$-- ========================================================================== -- + +// Import the local package and specific functions/variables needed +#import "@local/pandoc-cv:0.1.0": setup-style, horizontalrule, bodywidth, body-side, company-location, event-date, profile-photo, hidden-heading // Construct datetime object from structured date if provided via YAML map, else use today +// TODO: Move date logic into style.typ (see user feedback) #let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ -// Call the function from `style.typ` and pass variables to set up the document style -#show: style.setup-style.with( - $if(title)$title: "$title$", $endif$ // Comma after if present - $if(author)$author: "$author$", $endif$ // Comma after if present - $if(email)$email: "$email$".replace("\\", ""), $endif$ // Comma after if present - $if(github)$github: "$github$", $endif$ // Comma after if present - $if(gitlab)$gitlab: "$gitlab$", $endif$ // Comma after if present - $if(linkedin)$linkedin: "$linkedin$", $endif$ // Comma after if present - $if(website)$website: "$website$", $endif$ // Comma after if present - date: date-object, // Always present, comma needed before next potential arg - $if(keywords)$keywords: ($for(keywords)$ "$keywords$", $endfor$), $endif$ // Comma after if present - hyphenate: auto // Last argument, no trailing comma +// Call the setup function from the imported package using original conditional logic +#show: setup-style.with( + $if(title)$title: "$title$", $endif$ + $if(author)$author: "$author$", $endif$ + $if(email)$email: "$email$".replace("\\", ""), $endif$ + $if(github)$github: "$github$", $endif$ + $if(gitlab)$gitlab: "$gitlab$", $endif$ + $if(linkedin)$linkedin: "$linkedin$", $endif$ + $if(website)$website: "$website$", $endif$ + date: date-object, // Always present + $if(keywords)$keywords: ($for(keywords)$ "$keywords$", $endfor$), $endif$ + hyphenate: $if(hyphenate)$$hyphenate$$else$auto$endif$ // Use original logic ) -#block(width: style.bodywidth)[ +#block(width: bodywidth)[ $body$ diff --git a/typst-letter.typ b/typst-letter.typ index 1abd0ab..486c9eb 100644 --- a/typst-letter.typ +++ b/typst-letter.typ @@ -1,4 +1,14 @@ -#import "style.typ" // Import the style module +$-- ========================================================================== -- +$-- WARNING: PANDOC TEMPLATE - NOT PURE TYPST -- +$-- ========================================================================== -- +$-- This file uses Pandoc's templating syntax (e.g., $variable$, $if(...)$). -- +$-- It will NOT pass Typst linting or compilation directly. -- +$-- Pandoc processes this template *before* passing the result to Typst. -- +$-- Ignore Typst linter errors related to Pandoc syntax in this file. -- +$-- ========================================================================== -- + +// Import the local package and specific functions/variables needed +#import "@local/pandoc-cv:0.1.0": setup-style, horizontalrule, bodywidth // Construct datetime object from structured date if provided via YAML map, else use today #let date-object = $if(date.year)$datetime(year: $date.year$, month: $date.month$, day: $date.day$)$else$datetime.today()$endif$ @@ -17,7 +27,7 @@ } // Call the function from `style.typ` and pass variables to set up the document style -#show: style.setup-style.with( +#show: setup-style.with( $if(title)$title: "$title$", $endif$ // Comma after if present $if(author)$author: "$author$", $endif$ // Comma after if present $if(email)$email: "$email$".replace("\\", ""), $endif$ // Comma after if present @@ -49,13 +59,13 @@ box(height: intro-height)[#from-content] h(1fr) - box(height: intro-height, width: style.bodywidth)[#to-content] + box(height: intro-height, width: bodywidth)[#to-content] } #v(1em) #align(right)[ - #block(width: style.bodywidth)[ + #block(width: bodywidth)[ #set align(left) $body$ From d05f3df043d415a5d1a072f935a3d5f8a2666a90 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 13:29:55 +0000 Subject: [PATCH 27/63] Add Ubuntu-based Dockerfile and update devcontainer configuration for improved development setup --- .devcontainer/Dockerfile.ubuntu | 86 +++++++++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 15 +++--- 2 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 .devcontainer/Dockerfile.ubuntu diff --git a/.devcontainer/Dockerfile.ubuntu b/.devcontainer/Dockerfile.ubuntu new file mode 100644 index 0000000..8b6ee55 --- /dev/null +++ b/.devcontainer/Dockerfile.ubuntu @@ -0,0 +1,86 @@ +# Use a Microsoft base image with common dev tools pre-installed +FROM mcr.microsoft.com/devcontainers/base:ubuntu-24.04 + +# Install system dependencies using apt-get +# Need: git, pandoc, typst (manual?), shellcheck, just (manual?), poppler-utils, +# python3, pipx (for uv, pre-commit), nodejs, npm, fontconfig, ruby, build-essential (for fontist) +USER root + +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends \ + git \ + pandoc \ + shellcheck \ + poppler-utils \ + python3 \ + python3-pip \ + pipx \ + pre-commit \ + nodejs \ + npm \ + fontconfig \ + ruby \ + ruby-dev \ + build-essential \ + wget \ + ca-certificates \ + unzip \ + just \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Just is installed via apt-get above + +# --- Install Font Awesome 6 --- +RUN export FA_VERSION=6.7.2 \ + && export FA_DIR=/usr/local/share/fonts/fontawesome6 \ + && mkdir -p ${FA_DIR} \ + && wget -qO /tmp/fontawesome.zip https://github.com/FortAwesome/Font-Awesome/releases/download/${FA_VERSION}/fontawesome-free-${FA_VERSION}-desktop.zip \ + && unzip -q /tmp/fontawesome.zip -d /tmp/fontawesome-extract 'fontawesome-free-*-desktop/otfs/*' \ + && mv /tmp/fontawesome-extract/fontawesome-free-*-desktop/otfs/*.otf ${FA_DIR}/ \ + && fc-cache -f -v \ + && rm -rf /tmp/fontawesome.zip /tmp/fontawesome-extract + +# Install Typst manually (using musl build might work, or check for glibc build) +# Let's try the musl build first as it's known +ARG TYPST_VERSION=v0.12.0 +ARG TARGET=x86_64-unknown-linux-musl +RUN wget -qO typst.tar.xz https://github.com/typst/typst/releases/download/${TYPST_VERSION}/typst-${TARGET}.tar.xz \ + && tar xf typst.tar.xz \ + && mv typst-${TARGET}/typst /usr/local/bin/typst \ + && chmod +x /usr/local/bin/typst \ + && rm -rf typst.tar.xz typst-${TARGET} \ + && typst --version + +# Install fontist +RUN gem install fontist --no-document --version "~> 1.21.1" + +# Switch back to vscode user provided by base image +USER vscode + +# Install fonts using fontist (run as vscode user) +COPY --chown=vscode:vscode fontist-manifest.yml /tmp/fontist-manifest.yml +RUN fontist update --quiet \ + && fontist manifest-install --quiet /tmp/fontist-manifest.yml \ + && fontist cache clear --quiet \ + && rm /tmp/fontist-manifest.yml \ + && sudo fc-cache -fv # Update system font cache + +# --- Setup Local Typst Package Symlink for Development --- +# Define the target package path using the standard ENV var name +# Use the user's local share directory now +ENV TYPST_PACKAGE_PATH=/home/vscode/.local/share/typst/packages +ENV DEV_PKG_DIR=$TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0 +# Create the directory structure +RUN mkdir -p $DEV_PKG_DIR +# Create symlinks from the workspace files to the package directory +# This runs post-checkout, so workspace files exist +# Use a postCreateCommand in devcontainer.json instead of RUN here + +# Set Font Path (includes fontist user dir and system FA6 dir) +ENV TYPST_FONT_PATHS=/home/vscode/.fontist/fonts:/usr/local/share/fonts/fontawesome6 + +# Ensure pipx path is in PATH +ENV PATH="/home/vscode/.local/bin:${PATH}" + +RUN mkdir -p /home/vscode/.vscode-server/data/User/globalStorage diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 50c0abf..b82f328 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { - "name": "${localWorkspaceFolderBasename}-devcontainer", + "name": "${localWorkspaceFolderBasename}-devcontainer-ubuntu", "build": { - "dockerfile": "./Dockerfile", + "dockerfile": "./Dockerfile.ubuntu", "context": ".." }, "runArgs": [ @@ -12,9 +12,12 @@ "ghcr.io/devcontainers/features/common-utils:2": { "username": "vscode" }, - "ghcr.io/cirolosapio/devcontainers-features/alpine-docker-outside-of-docker:0": {} + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "latest", + "moby": true + } }, - "postCreateCommand": "sudo chmod 666 /var/run/docker.sock && pre-commit install", + "postCreateCommand": "mkdir -p /home/vscode/.local/share/typst/packages/local/pandoc-cv/0.1.0 && ln -sf /workspaces/typstCV/style.typ /home/vscode/.local/share/typst/packages/local/pandoc-cv/0.1.0/style.typ && ln -sf /workspaces/typstCV/typst.toml /home/vscode/.local/share/typst/packages/local/pandoc-cv/0.1.0/typst.toml && pre-commit install", "remoteUser": "vscode", "customizations": { "vscode": { @@ -32,8 +35,8 @@ }, "mounts": [ { - "source": "${localWorkspaceFolderBasename}-data", - "target": "/home/vscode", + "source": "${localWorkspaceFolderBasename}-vscode-global-storage", + "target": "/home/vscode/.vscode-server/data/User/globalStorage", "type": "volume" } ] From 1f1986128946b24417f3dfa9d3b9790059da483c Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:50:15 +0000 Subject: [PATCH 28/63] Add 'file' package to Dockerfile for improved dependency management --- .devcontainer/Dockerfile.ubuntu | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile.ubuntu b/.devcontainer/Dockerfile.ubuntu index 8b6ee55..b514996 100644 --- a/.devcontainer/Dockerfile.ubuntu +++ b/.devcontainer/Dockerfile.ubuntu @@ -9,6 +9,7 @@ USER root RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get install -y --no-install-recommends \ git \ + file \ pandoc \ shellcheck \ poppler-utils \ From 6f08942026a52dd40370180e7da651e5205911c2 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:52:35 +0000 Subject: [PATCH 29/63] Refactor Dockerfile to improve font installation and Typst setup, streamline build stages, and enhance project file organization --- Dockerfile | 122 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 42 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6f1811..127cbaf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,56 +1,94 @@ -ARG FEDORA_VERSION=41 -ARG ALPINE_VERSION=3.20 -ARG TYPST_FONTAWESOME_VERSION=0.5.0 - -FROM alpine:${ALPINE_VERSION} AS builder +# === BUILDER STAGE: Install fonts with fontist === +FROM ruby:3.2-slim AS fonts-builder +RUN apt-get update && apt-get install -y --no-install-recommends \ + git build-essential \ + && rm -rf /var/lib/apt/lists/* +RUN gem install fontist --no-document --version "~> 1.21.1" +COPY fontist-manifest.yml . +RUN fontist update --quiet \ + && fontist manifest-install --quiet fontist-manifest.yml \ + && fontist cache clear --quiet \ + && chmod -R 644 /root/.fontist/fonts +# === BUILDER STAGE: Install Typst and packages === +FROM alpine:3.21 AS typst-builder +# Install dependencies needed for download (wget), extraction (tar with xz support), and git (for packages) +RUN apk add --no-cache wget tar xz git +# Install Typst v0.12.0 manually ARG TYPST_VERSION=v0.12.0 +ARG TARGET=x86_64-unknown-linux-musl +RUN wget -qO typst.tar.xz https://github.com/typst/typst/releases/download/${TYPST_VERSION}/typst-${TARGET}.tar.xz \ + && tar xf typst.tar.xz \ + && mv typst-${TARGET}/typst /usr/local/bin/typst \ + && chmod +x /usr/local/bin/typst \ + && rm -rf typst.tar.xz typst-${TARGET} \ + && typst --version +# Download and extract typst-fontawesome v0.5.0 manually +ARG TYPST_FONTAWESOME_VERSION=0.5.0 +ENV TYPST_PACKAGE_PATH=/root/.local/share/typst/packages +RUN mkdir -p $TYPST_PACKAGE_PATH/preview/fontawesome/${TYPST_FONTAWESOME_VERSION}/ \ + && wget -qO typst-fontawesome.tar.gz https://packages.typst.org/preview/fontawesome-${TYPST_FONTAWESOME_VERSION}.tar.gz \ + && tar xf typst-fontawesome.tar.gz -C $TYPST_PACKAGE_PATH/preview/fontawesome/${TYPST_FONTAWESOME_VERSION}/ \ + && rm typst-fontawesome.tar.gz -RUN wget -qO typst.tar.xz https://github.com/typst/typst/releases/download/${TYPST_VERSION}/typst-x86_64-unknown-linux-musl.tar.xz \ - && tar xf typst.tar.xz - -ARG TYPST_FONTAWESOME_VERSION -RUN wget -qO typst-fontawesome.tar.gz https://packages.typst.org/preview/fontawesome-${TYPST_FONTAWESOME_VERSION}.tar.gz \ - && mkdir /fontawesome \ - && tar xf typst-fontawesome.tar.gz -C /fontawesome - -FROM fedora:${FEDORA_VERSION} - -ARG PANDOC_VERSION=3.1.11.1 -ARG JUST_VERSION=1.35.0 -ARG FIRA_SANS_VERSION=4.202 -ARG IBM_PLEX_VERSION=6.4.0 -ARG FONTAWESOME_VERSION=6.6.0 -RUN dnf update -yq \ - && dnf install -yq \ - just-${JUST_VERSION} \ - pandoc-${PANDOC_VERSION} \ - mozilla-fira-sans-fonts-${FIRA_SANS_VERSION} \ - ibm-plex-serif-fonts-${IBM_PLEX_VERSION} \ - fontawesome-6-brands-fonts-${FONTAWESOME_VERSION} \ - fontawesome-6-free-fonts-${FONTAWESOME_VERSION} \ - poppler-utils \ - && dnf clean all - -ENV PANDOC_DATA_DIR=/usr/share/pandoc-${PANDOC_VERSION}/ +# === FINAL STAGE: Alpine Production Image === +FROM alpine:3.21 -COPY typst-cv.typ typst-letter.typ ${PANDOC_DATA_DIR}/data/templates/ -COPY *.lua ${PANDOC_DATA_DIR}/filters/ +# Install runtime dependencies: bash, pandoc, fontconfig, font-awesome +# Git is NOT needed here anymore as packages are copied from builder +# wget/tar are NOT needed here anymore as Typst is copied from builder +RUN apk add --no-cache \ + bash \ + pandoc \ + fontconfig \ + font-awesome \ + font-awesome-free \ + font-awesome-brands \ + && rm -rf /var/cache/apk/* -COPY --from=builder typst-x86_64-unknown-linux-musl/typst /usr/bin/typst +# --- Copy Assets from Builders --- +# Copy Typst binary +COPY --from=typst-builder /usr/local/bin/typst /usr/local/bin/typst +# Copy Typst packages (fontawesome) +COPY --from=typst-builder /root/.local/share/typst/packages /usr/share/typst/packages +# Copy fonts +COPY --from=fonts-builder /root/.fontist/fonts/ /usr/share/fonts/fontist/ -ARG PANDOC_CV_VERSION=0.1.0 -ENV TYPST_PACKAGE_PATH=/usr/local/share/typst/packages/ -COPY style.typ typst.toml ${TYPST_PACKAGE_PATH}/local/pandoc-cv/${PANDOC_CV_VERSION}/ +# --- Configure Environment --- +# Define paths +ENV PANDOC_DATA_DIR=/usr/share/pandoc +ENV TYPST_PACKAGE_PATH=/usr/share/typst/packages +# Reset font paths to only include font directories +ENV TYPST_FONT_PATHS=/usr/share/fonts -ARG TYPST_FONTAWESOME_VERSION -COPY --from=builder /fontawesome/*.typ /fontawesome/typst.toml ${TYPST_PACKAGE_PATH}/preview/fontawesome/${TYPST_FONTAWESOME_VERSION}/ +# Create necessary directories (including the local package structure) +RUN mkdir -p $PANDOC_DATA_DIR/filters \ + $PANDOC_DATA_DIR/templates \ + $TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0 \ + /data -ENV TYPST_FONT_PATHS=/usr/share/fonts/ +# Update font cache AFTER copying fonts and setting path +RUN fc-cache -fv +# --- Copy Project Files --- COPY build.sh /usr/local/bin/build.sh +COPY linkify.lua $PANDOC_DATA_DIR/filters/linkify.lua +COPY typst-cv.lua $PANDOC_DATA_DIR/filters/typst-cv.lua +COPY typst-cv.typ $PANDOC_DATA_DIR/templates/typst-cv.typ +COPY typst-letter.typ $PANDOC_DATA_DIR/templates/typst-letter.typ +# Copy style.typ and typst.toml to the local package directory +COPY style.typ $TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0/style.typ +COPY typst.toml $TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0/typst.toml + +# Ensure build.sh is executable RUN chmod +x /usr/local/bin/build.sh +# --- Final Setup --- +# Set working directory WORKDIR /data -ENTRYPOINT [ "build.sh" ] +# Set entrypoint +ENTRYPOINT ["build.sh"] + +# Default command (optional, can be overridden) +CMD ["--help"] From f5e2db9e78e3ddaccff44f91fbc4fb62a8675fdc Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:55:32 +0000 Subject: [PATCH 30/63] Enhance build.sh to handle stdin input, enforce single input file restriction, and improve output path logic for better error handling and clarity --- build.sh | 106 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index 3c1050e..2a59ddd 100755 --- a/build.sh +++ b/build.sh @@ -30,6 +30,11 @@ usage() { # --- Argument Parsing --- while [ $# -gt 0 ]; do case "$1" in + -) # Handle stdin explicitly + if [ -n "$input_file" ]; then echo "Error: Only one input file allowed ('-' or filename)." >&2; usage; fi + input_file="-" + shift + ;; --type) if [ -z "$2" ]; then echo "Error: --type requires an argument." >&2; usage; fi if [ "$2" != "cv" ] && [ "$2" != "letter" ]; then echo "Error: --type must be 'cv' or 'letter'." >&2; usage; fi @@ -115,6 +120,12 @@ fi template_file="typst-${doc_type}.typ" echo "Info: Using document type: $doc_type (template: $template_file)" >&2 +# Force override pipeline for CVs to handle potential photo paths correctly via input_dir_abs +if [ "$doc_type" = "cv" ]; then + echo "Info: Forcing override pipeline for CV to handle potential relative image paths." >&2 + use_override_pipeline=true +fi + # --- Determine Output Path --- # Reverted: Simplified output path logic output_arg="" # Will be '-o path' or '-' for stdout @@ -129,14 +140,19 @@ elif [ -n "$output_file" ]; then output_path="${output_dir}/${output_file}" output_arg="-o $output_path" else - # Default output filename based on input - if [ "$input_file" = "-" ]; then - echo "Error: Output file must be specified when reading from stdin (unless outputting to stdout with '--output -')." >&2 - usage + # Default output filename based on input (only if input is not stdin) + if [ "$input_file" != "-" ]; then + base_name=$(basename "$input_file" .md) + output_path="${output_dir}/${base_name}.pdf" # Always .pdf now + output_arg="-o $output_path" + # If input is stdin, output MUST be stdout ('-') + elif [ "$input_file" = "-" ] && [ "$output_file" != "-" ]; then + # This case is only reached if input is '-' and output was NOT specified as '-' OR was empty + echo "Error: Output must be explicitly specified as stdout ('--output -') when reading from stdin ('-')." >&2 + usage fi - base_name=$(basename "$input_file" .md) - output_path="${output_dir}/${base_name}.pdf" # Always .pdf now - output_arg="-o $output_path" + # If input is stdin and output is stdout, output_arg remains "-" + # If input is file and output is file, output_arg is "-o path" fi # Create output directory if needed and not outputting to stdout @@ -150,9 +166,41 @@ fi # --- Build Commands --- # Check if PANDOC_DATA_DIR is set, provide default if not (useful if run outside Docker) -PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-/usr/share/pandoc}" +# Ensure PANDOC_DATA_DIR is exported so subshells (like in pipes) might see it, though direct arg is better. +export PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-/usr/share/pandoc}" +# Determine resource path (input file's directory or '.' for stdin) +resource_path="." +if [ "$input_file" != "-" ]; then + # Handle potential edge case where input file is in root (dirname returns '.') + dir=$(dirname "$input_file") + if [ "$dir" != "." ]; then + resource_path="$dir" + fi +fi +# Add --data-dir to the base command - resource path added per-command # shellcheck disable=SC2089 -pandoc_base="pandoc --data-dir=\"$PANDOC_DATA_DIR\" --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" +pandoc_base="pandoc --data-dir $PANDOC_DATA_DIR --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" + +# --- Calculate Absolute Input Directory for Typst --- +input_dir_abs="" +if [ "$input_file" != "-" ]; then + # Get absolute path of the input file + # Handle relative paths by prepending PWD if needed + case "$input_file" in + /*) input_file_abs="$input_file" ;; + *) input_file_abs="$PWD/$input_file" ;; + esac + # Get the directory part + input_dir_abs=$(dirname "$input_file_abs") + # Add to typst args if not empty (should always be set if input is a file) + if [ -n "$input_dir_abs" ]; then + # shellcheck disable=SC2089 + typst_input_args="$typst_input_args --input input_dir_abs=\"$input_dir_abs\"" + # Note: This variable is ONLY used by the override pipeline's typst compile step. + # The direct pandoc pipeline cannot use it. + fi +fi + # --- Execute Pipeline --- # Reverted: Simplified execution logic back to original override check @@ -161,15 +209,45 @@ if [ "$use_override_pipeline" = true ]; then echo "Info: Using override pipeline (Pandoc -> Typst -> PDF)." >&2 # Construct the pandoc part of the pipe pandoc_cmd_part="$pandoc_base --to=typst --template=$template_file $input_arg" + + # Prepare typst output argument. + # Typst compile takes the output path directly as the last argument, + # or '-' for stdout. It does NOT use '-o'. + typst_output_arg="" + if [ "$output_arg" = "-" ]; then + typst_output_arg="-" # Pass '-' for stdout + elif echo "$output_arg" | grep -q -e '^-o '; then + # Extract path after '-o ' for the argument + typst_output_arg=$(echo "$output_arg" | sed 's/^-o //') + fi + + # Determine the root directory for Typst. Use absolute path if available, else default to /data for stdin. + typst_root_dir="/data" # Default for stdin + if [ -n "$input_dir_abs" ]; then + typst_root_dir="$input_dir_abs" + fi + # Execute pandoc part directly and pipe to typst compile - # shellcheck disable=SC2086,SC2090 # We want word splitting; quotes in var are for target cmd - $pandoc_cmd_part | typst compile $typst_input_args - $output_arg + # Add --root argument. Pass calculated output argument ($typst_output_arg) directly to typst. + # shellcheck disable=SC2086,SC2090 # We want word splitting for $typst_input_args; quotes are handled there. $typst_output_arg is single path or '-'. + pandoc_pipe_cmd="$pandoc_cmd_part | typst compile --root \"$typst_root_dir\" $typst_input_args - $typst_output_arg" + + # Execute the combined pipeline command using sh -c for potentially better handling of pipes/quotes. + sh -c "$pandoc_pipe_cmd" + # Check exit status of the pipe (specifically the last command, typst compile) + pipe_status=$? + if [ $pipe_status -ne 0 ]; then + echo "Error: Typst compilation failed with status $pipe_status." >&2 + exit $pipe_status + fi + else echo "Info: Using direct pipeline (Pandoc -> PDF)." >&2 - # Construct the full pandoc command parts - pandoc_cmd_part1="$pandoc_base --template=$template_file" + # Construct the full pandoc command parts, adding resource path here + # shellcheck disable=SC2089 + pandoc_cmd_part1="$pandoc_base --resource-path \"$resource_path\" --template=$template_file" pandoc_cmd_part2="$input_arg" - # $output_arg contains '-o path' or is empty if default CWD output + # $output_arg contains '-o path' or '-' for stdout # Execute directly, let shell handle splitting of $output_arg # shellcheck disable=SC2086,SC2090 # We want word splitting; quotes in var are for target cmd $pandoc_cmd_part1 "$pandoc_cmd_part2" $output_arg From fdd606a104f7f77444b7b52be14770cfb9cc227d Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:57:05 +0000 Subject: [PATCH 31/63] Add Bats tests for Docker image and update CV templates for consistency --- tests/docker.bats | 113 +++++++++++++++++++++++ tests/filter/filters.bats | 10 +- tests/fixtures/example-cv-no-optional.md | 2 +- tests/fixtures/example-cv-stdin.md | 16 ++++ tests/fixtures/example-cv.md | 2 +- tests/fixtures/typst_cv_filter_test.md | 4 +- 6 files changed, 138 insertions(+), 9 deletions(-) create mode 100755 tests/docker.bats create mode 100644 tests/fixtures/example-cv-stdin.md diff --git a/tests/docker.bats b/tests/docker.bats new file mode 100755 index 0000000..8d9699d --- /dev/null +++ b/tests/docker.bats @@ -0,0 +1,113 @@ +#!/usr/bin/env bats + +load 'test_helper/bats-support/load' +load 'test_helper/bats-assert/load' + +# --- Variables --- +# Assumes the image is built with the 'latest' tag locally +DOCKER_IMAGE="typst-cv:latest" +FIXTURES_DIR="tests/fixtures" # Relative path inside the devcontainer +OUTPUT_DIR_TMP="" + +# --- Setup & Teardown --- +setup() { + # Ensure the image exists locally + docker image inspect "$DOCKER_IMAGE" > /dev/null 2>&1 || { + echo "!!! Docker image '$DOCKER_IMAGE' not found. Build it first (e.g., 'just build-docker')." >&3 + return 1 + } + # Create a temporary output directory for this test run + OUTPUT_DIR_TMP=$(mktemp -d) +} + +teardown() { + # Clean up temporary output directory + if [ -n "$OUTPUT_DIR_TMP" ]; then + rm -rf "$OUTPUT_DIR_TMP" + fi +} + +# --- Test Cases --- + +@test "docker run (no args): shows usage message" { + # Run without arguments, expect usage message and non-zero exit code + run docker run --rm "$DOCKER_IMAGE" + assert_failure # Script should exit with error if no input given + assert_output --partial "Usage: /usr/local/bin/build.sh" +} + +@test "docker run : generates PDF output" { + local input_file="$FIXTURES_DIR/example-cv.md" + local output_file="$OUTPUT_DIR_TMP/example-cv.pdf" + + # Mount fixtures read-only using host path (should work with DinD), mount output dir writable + run docker run --rm \ + -v "$(pwd)/${FIXTURES_DIR}:/test-fixtures:ro" \ + -v "$OUTPUT_DIR_TMP:/output" \ + "$DOCKER_IMAGE" --output-dir /output /test-fixtures/example-cv.md + + assert_success + assert [ -f "$output_file" ] + # Basic check: is it a PDF file? + run file "$output_file" + assert_output --partial "PDF document" +} + +@test "docker run --type letter: generates letter PDF" { + local input_file="$FIXTURES_DIR/example-letter.md" + local output_file="$OUTPUT_DIR_TMP/example-letter.pdf" + + run docker run --rm \ + -v "$(pwd)/${FIXTURES_DIR}:/test-fixtures:ro" \ + -v "$OUTPUT_DIR_TMP:/output" \ + "$DOCKER_IMAGE" --output-dir /output --type letter /test-fixtures/example-letter.md + + assert_success + assert [ -f "$output_file" ] + run file "$output_file" + assert_output --partial "PDF document" +} + +@test "docker run stdin > stdout: pipes PDF" { + local input_file="$FIXTURES_DIR/example-cv-stdin.md" # Use the new minimal fixture + local output_file="$OUTPUT_DIR_TMP/stdout.pdf" + + # Pipe input file to docker run, redirect output to file + # Mount fixtures read-only using host path (though not strictly needed for this fixture) + # Pass '-' to build.sh to indicate reading from stdin + run bash -c "cat \"./$input_file\" | docker run --rm -i \ + -v \"$(pwd)/${FIXTURES_DIR}:/test-fixtures:ro\" \ + \"$DOCKER_IMAGE\" - --output - > \"$output_file\"" # Note: -i is needed for stdin, '-' tells build.sh to use stdin + + assert_success + assert [ -f "$output_file" ] + assert [ -s "$output_file" ] # Check if file is not empty + run file "$output_file" + assert_output --partial "PDF document" +} + +@test "docker run --set: overrides metadata" { + local input_file="$FIXTURES_DIR/example-cv.md" + local output_file="$OUTPUT_DIR_TMP/override.pdf" + local override_author="Test Override Author" # Changed variable name for clarity + + # Explicitly set the output filename within the container + # Use --set author=... to match the variable used in YAML/template + run docker run --rm \ + -v "$(pwd)/${FIXTURES_DIR}:/test-fixtures:ro" \ + -v "$OUTPUT_DIR_TMP:/output" \ + "$DOCKER_IMAGE" --output-dir /output --output override.pdf --set author="$override_author" /test-fixtures/example-cv.md + + assert_success + # Check the expected file exists on the host via the volume mount + assert [ -f "$output_file" ] + + # Requires pdftotext (should be available on host if devcontainer is used, or install locally) + # This check is basic, assumes pdftotext is available + if command -v pdftotext &> /dev/null; then + run pdftotext "$output_file" - + assert_output --partial "$override_author" # Check for the correct override value + else + skip "pdftotext not found, skipping content check" + fi +} diff --git a/tests/filter/filters.bats b/tests/filter/filters.bats index 05c18f9..80b3bc2 100755 --- a/tests/filter/filters.bats +++ b/tests/filter/filters.bats @@ -60,7 +60,7 @@ setup() { local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" run eval "$cmd" assert_success - assert_output --partial 'side: style.profile-photo(image("photo.png", width: 100pt))' + assert_output --partial 'side: profile-photo(image("photo.png", width: 100pt))' } @test "filter/typst-cv: handles hidden class" { @@ -68,7 +68,7 @@ setup() { local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" run eval "$cmd" assert_success - assert_output --partial '#style.hidden-heading()' + assert_output --partial '#hidden-heading()' } @test "filter/typst-cv: handles location attribute" { @@ -77,7 +77,7 @@ setup() { run eval "$cmd" assert_success # Check for the specific function call and the content (actual output has single backslash) - assert_output --partial 'side: style.event-date()[City \ Country]' + assert_output --partial 'side: event-date()[City \ Country]' } @test "filter/typst-cv: handles date attribute" { @@ -85,7 +85,7 @@ setup() { local cmd="$PANDOC_BASE_CMD --template=typst-cv.typ --lua-filter=typst-cv.lua \"$fixture\"" run eval "$cmd" assert_success - assert_output --partial 'side: style.company-location()[2023 - Present]' + assert_output --partial 'side: company-location()[2023 - Present]' } @test "filter/typst-cv: handles ordered list" { @@ -102,7 +102,7 @@ setup() { run eval "$cmd" assert_success # Check date was processed (include closing parenthesis) - assert_output --partial 'side: style.company-location()[Date First])' + assert_output --partial 'side: company-location()[Date First])' # Check location was NOT processed on the same header refute_output --partial 'Header With Both.*side: event-date()' # Keep .* here for refute flexibility } diff --git a/tests/fixtures/example-cv-no-optional.md b/tests/fixtures/example-cv-no-optional.md index a3e3154..a418320 100644 --- a/tests/fixtures/example-cv-no-optional.md +++ b/tests/fixtures/example-cv-no-optional.md @@ -13,7 +13,7 @@ links: Pandoc: https://pandoc.org/ --- -# Jane Doe Minimal {photo='image("tests/fixtures/placeholder-photo.png", width: 120pt)'} +# Jane Doe Minimal {photo="placeholder-photo.png"} ## Summary diff --git a/tests/fixtures/example-cv-stdin.md b/tests/fixtures/example-cv-stdin.md new file mode 100644 index 0000000..61b86d7 --- /dev/null +++ b/tests/fixtures/example-cv-stdin.md @@ -0,0 +1,16 @@ +--- +title: Minimal CV for Stdin Test +author: Stdin User +email: stdin@example.com +--- + +# Stdin User + +## Summary + +This is a minimal CV used for testing stdin piping in Docker tests. It does not contain images or complex formatting. + +## Skills + +- Testing +- Stdin diff --git a/tests/fixtures/example-cv.md b/tests/fixtures/example-cv.md index f5d95b7..2546472 100644 --- a/tests/fixtures/example-cv.md +++ b/tests/fixtures/example-cv.md @@ -19,7 +19,7 @@ links: private_info: This is private info from YAML --- -# Jane Doe {photo='image("tests/fixtures/placeholder-photo.png", width: 120pt)'} +# Jane Doe {photo="placeholder-photo.png"} ## Summary diff --git a/tests/fixtures/typst_cv_filter_test.md b/tests/fixtures/typst_cv_filter_test.md index 43b4d5a..2cec496 100644 --- a/tests/fixtures/typst_cv_filter_test.md +++ b/tests/fixtures/typst_cv_filter_test.md @@ -2,8 +2,8 @@ title: Typst CV Filter Test - Current Logic --- -# Name {photo='image("photo.png", width: 100pt)'} - +# Name {photo="photo.png" photowidth="100pt"} + This header has a photo attribute. ## Section 1 {.hidden} From 3ef3237c0133e66b362d30de270ff5e939cfa1bf Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:57:39 +0000 Subject: [PATCH 32/63] Enhance Header function to prioritize 'photo' attribute handling and streamline image insertion logic --- typst-cv.lua | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/typst-cv.lua b/typst-cv.lua index aac5020..2e02eda 100644 --- a/typst-cv.lua +++ b/typst-cv.lua @@ -18,16 +18,33 @@ function Header(el) table.insert(typst_blocks, el) table.insert(typst_blocks, pandoc.RawBlock("typst", string.format("], side: event-date()[%s])", value))) return typst_blocks + -- New logic: Check for 'photo' attribute first elseif key == "photo" then + local photo_path = value + local photo_width = el.attributes.photowidth or "120pt" -- Get width or use default + + -- No need to calculate absolute path here. + -- Typst's --root argument (set in build.sh) handles resolving relative paths. + + -- Create the image object string using the path from the attribute and the width + local image_arg = string.format('image("%s", width: %s)', photo_path, photo_width) + + -- Insert the Typst blocks table.insert(typst_blocks, pandoc.RawBlock("typst", "#body-side([")) table.insert(typst_blocks, el) table.insert(typst_blocks, - pandoc.RawBlock("typst", string.format("], side: profile-photo(%s))", value))) + pandoc.RawBlock("typst", string.format("], side: profile-photo(%s))", image_arg))) return typst_blocks + -- Ignore 'photowidth' key here as it's handled when 'photo' is found + elseif key == "photowidth" then + -- Do nothing, processed with 'photo' end - end - end + -- Keep handling other keys if necessary (add more elseifs here) + + end -- End of attribute loop + end -- End of if el.attributes + -- Handle hidden class separately, after attribute processing if el.classes:includes("hidden") then local typst_blocks = {} table.insert(typst_blocks, pandoc.RawBlock("typst", "#hidden-heading()[")) From 515087dc494f84e1491d67de73896f2ef75fa205 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:58:00 +0000 Subject: [PATCH 33/63] Add Docker usage tests to the Justfile and update test task to include them --- justfile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index 12ce525..0153489 100644 --- a/justfile +++ b/justfile @@ -26,8 +26,12 @@ test-filter: test-e2e: sh {{e2e_tests_script}} -# Run all tests -test: test-unit test-filter test-e2e +# Run Docker usage tests (requires image built, e.g., via `just build-docker`) +test-docker: + {{bats_executable}} tests/docker.bats + +# Run all tests (including Docker tests) +test: test-unit test-filter test-e2e test-docker @echo "All tests passed!" # Clean temporary files (if any - currently handled by tests) @@ -39,3 +43,11 @@ build-examples: ./build.sh --output-dir examples tests/fixtures/example-cv.md ./build.sh --output-dir examples tests/fixtures/example-letter.md --type letter @echo "Example PDFs built in ./examples/" + +# Build the default Docker image (Alpine-based) +build-docker tag='latest': + docker build --pull -t typst-cv:{{tag}} -f Dockerfile . + +# Build the Fedora-based Docker image +build-docker-fedora tag='latest-fedora': + docker build --pull -t typst-cv:{{tag}} -f Dockerfile.fedora . From 9263bd6b0cbb619812e52271a442047d445db382 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:58:13 +0000 Subject: [PATCH 34/63] Enhance CI workflow to include Docker usage tests and adjust test execution paths for improved clarity and reliability --- .github/workflows/ci.yml | 53 ++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8775bb..05997fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,34 +66,49 @@ jobs: cache-from: type=registry,ref=${{ env.IMAGE }}:testing # Cache from previous testing tag if exists cache-to: type=inline - - name: Checkout code + - name: Checkout code (including submodules for bats) uses: actions/checkout@v4 + with: + submodules: recursive # Needed to get bats-core for docker tests + + - name: Install Bats and File command (for Docker usage tests) + run: sudo apt-get update && sudo apt-get install -y bats file - - name: Run tests inside the container + - name: Run tests inside the container (using production image, mounting only fixtures) run: | echo "Running tests..." - # Execute test suites - docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - bats ${{ env.TEST_DIR }}/unit/build_sh.bats + # Mount only fixtures to /test-fixtures, run tests relative to /data (workdir) + # Note: Tests might need adjustment if they expect to be run from project root + # Assuming bats and sh scripts handle paths correctly when run from /data + docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + bats /test-fixtures/../unit/build_sh.bats # Adjust path relative to mount point if needed - docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - bats ${{ env.TEST_DIR }}/filter/filters.bats # Use bats for filter tests + docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + bats /test-fixtures/../filter/filters.bats # Adjust path relative to mount point if needed - # Removed execution of test_integration.sh - # docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - # sh ${{ env.TEST_DIR }}/test_integration.sh + docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + sh /test-fixtures/../test_e2e.sh # Adjust path relative to mount point if needed + echo "Container tests finished." - docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - sh ${{ env.TEST_DIR }}/test_e2e.sh - echo "Tests finished." - - - name: Build Example PDFs for Release Artifacts (after tests pass) + - name: Run Docker usage tests (on runner host) + run: | + echo "Running Docker usage tests..." + # These tests run on the host, interacting with the built testing image + # Assumes 'bats' is installed on the runner (installed in previous step) + # Assumes docker is available on the runner + bats ${{ env.TEST_DIR }}/docker.bats + echo "Docker usage tests finished." + + - name: Build Example PDFs for Release Artifacts (using production image, mounting only fixtures) run: | echo "Building example PDFs..." - docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - ./build.sh --output-dir examples ${{ env.TEST_DIR }}/fixtures/example-cv.md - docker run --rm -v "${PWD}:/data" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - ./build.sh --output-dir examples ${{ env.TEST_DIR }}/fixtures/example-letter.md --type letter + # Create host output directory first + mkdir -p ./examples + # Mount fixtures to /test-fixtures, mount host output dir to /output + docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" -v "${PWD}/examples:/output" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + build.sh --output-dir /output /test-fixtures/example-cv.md + docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" -v "${PWD}/examples:/output" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ + build.sh --output-dir /output /test-fixtures/example-letter.md --type letter echo "Example PDFs built." - name: Upload Example PDFs From e636f6231ca6ce7a70c32d6d62a4c01e6cc6712f Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 20:58:29 +0000 Subject: [PATCH 35/63] Refactor Docker setup and testing framework: switch to Ubuntu-based devcontainer, enhance Dockerfile for production image, and debug Docker usage tests for improved reliability. --- memory-bank/activeContext.md | 64 ++++++++++++++++++--------------- memory-bank/progress.md | 37 ++++++++++--------- memory-bank/techContext.md | 70 +++++++++++++++++++++++------------- 3 files changed, 102 insertions(+), 69 deletions(-) diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 0128cf0..25effe7 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,37 +1,45 @@ -# Active Context: Testing & Depersonalization Complete (2025-04-26) +# Active Context: Debugging Docker Tests & Switching Devcontainer (2025-04-28) ## Current Focus -Phase 1, focusing on adding tests, depersonalizing examples, and updating CI, is complete. The project now has a test suite covering unit, filter, and E2E scenarios, uses generic example files, and the CI workflow automatically runs these tests. - -## Recent Actions (Completed) - -- **Discussed Next Steps:** Prioritized adding tests and depersonalizing the project. -- **Test Setup:** - - Created `tests/` directory structure. - - Installed `bats-core` and helpers via Git submodules (`tests/bats`, `tests/test_helper`). -- **Depersonalization:** - - Created generic example files (`tests/fixtures/example-cv.md`, `tests/fixtures/example-letter.md`) using placeholder data and correct heading/link structure. - - Created placeholder image (`tests/fixtures/placeholder-photo.png`). - - Removed old personalized files (`kadykov-*.md`, `photo.jpg`). -- **Testing Implementation:** - - Added unit tests for `build.sh` using Bats (`tests/unit/build_sh.bats`), debugging path and execution issues. - - Refactored filter tests into Bats (`tests/filter/filters.bats`) using `grep`/`assert_output --partial` for robustness instead of fragile snapshots. - - Added E2E smoke tests (`tests/test_e2e.sh`) verifying successful PDF generation. Enhanced E2E tests using `pdftotext` (via `poppler-utils` added to Dockerfile) to check for presence of hidden headings and absence of optional footer links. Debugged various issues related to YAML format, paths, template logic, and test implementation. -- **Documentation:** Updated `README.md` to clarify heading structure, linkification, metadata override workflow, and point to new example files. -- **CI Update:** Modified `.github/workflows/ci.yml` to run the new test suites (unit, filter, e2e) inside the Docker container (which now includes `poppler-utils`) and removed the old GitHub Pages deployment job for CVs. Ensured example PDFs are built and uploaded for release artifacts. -- **Development Tooling:** Added `justfile` for convenient local execution of linters and test suites. -- **Memory Bank:** Updated `activeContext.md`, `progress.md`, `techContext.md`. +Debugging and fixing the Docker usage tests (`tests/docker.bats`) within the new Ubuntu-based Docker-in-Docker (DinD) devcontainer environment. + +## Recent Actions (This Session) + +- **Created Production Dockerfile:** Defined `Dockerfile` (Alpine-based) for the self-contained production image, including multi-stage builds, pinned Typst v0.12.0, and necessary dependencies/assets. +- **Updated CI:** Modified `.github/workflows/ci.yml` to build and test using the new production `Dockerfile`. +- **Added Docker Tests:** Created `tests/docker.bats` to test container interaction. +- **Updated `justfile`:** Added `build-docker` and `test-docker` recipes. +- **Refactored Package Handling:** + - Reverted templates (`typst-cv.typ`, `typst-letter.typ`) to use local package import (`@local/pandoc-cv:0.1.0`). + - Updated production `Dockerfile` to copy `style.typ` and `typst.toml` into the package structure. + - Updated devcontainer `Dockerfile` (`.devcontainer/Dockerfile.ubuntu`) to create symlinks for `style.typ` and `typst.toml` into the package structure. + - Added Pandoc comment syntax (`$--`) to template files to avoid linter issues. +- **Debugged `build.sh`:** Made several adjustments to handle Pandoc `--data-dir`, `--resource-path`, stdin (`-`) argument parsing, and `typst compile` arguments for the override pipeline. +- **Identified Devcontainer Issue:** Diagnosed Docker test failures as stemming from Docker-out-of-Docker limitations with host path mounts in the Alpine devcontainer. +- **Planned Devcontainer Switch:** + - Created a new Ubuntu-based devcontainer Dockerfile (`.devcontainer/Dockerfile.ubuntu`) with necessary tools. + - Updated `.devcontainer/devcontainer.json` to use the new Dockerfile and enable the Docker-in-Docker feature. + - Simplified `tests/docker.bats` to use direct host path mounts, anticipating the DinD setup. +- **Executed Tests:** Ran `just build-docker && just test` multiple times. +- **Fixed Path Resolution:** Modified `build.sh` and `typst-cv.lua` to correctly handle image paths relative to the input file using Typst's `--root` argument, resolving E2E test failures. +- **Refactored Photo Attribute:** Changed from `{photo='image(...)'}` to `{photo="path" photowidth="..."}` for better usability, updating `typst-cv.lua` and test fixtures (`*.md`) accordingly. +- **Fixed Filter Tests:** Made `typst-cv.lua` robust against missing metadata during filter tests. +- **Fixed Docker Tests:** + - Used an image-free fixture (`example-cv-stdin.md`) for the stdin test. + - Corrected the `--set` key from `name` to `author` in the override test. + - Ensured the override test explicitly requested the correct output filename (`--output override.pdf`). + - Investigated and resolved issues preventing the output file creation in the override test. ## Decisions & Notes -- Decided *not* to implement `--output-format typst` in `build.sh` at this time due to difficulties in reliably generating/testing the `.typ` output via the script. Integration testing relies on filter tests and E2E PDF tests. -- Noted suggestions for future filter (`typst-cv.lua`) refactoring (fixing function names, adding generic side content, multiple attributes, div support) but postponed implementation until after establishing baseline tests. -- Corrected understanding of YAML date format and relative paths for Typst compilation. -- **Bug Fix:** Corrected default values (`"default"` -> `none`) and added conditional logic in `style.typ` footer to prevent displaying icons/links for optional fields (github, gitlab, linkedin, website) if they are not provided in YAML or via overrides. -- **Template Imports:** Confirmed templates use direct `#import "style.typ"` as updated by user. +- Devcontainer successfully switched to Ubuntu (DinD), simplifying Docker testing. +- Photo handling now uses separate `photo` (path) and optional `photowidth` attributes in Markdown for improved user experience. +- Path resolution for images relies on Typst's `--root` argument, set correctly by `build.sh`. +- All test suites (unit, filter, E2E, docker) are now passing. ## Immediate Next Steps -- Update `progress.md`. -- Present the completed Phase 1 work (including bug fix) to the user. +- Update `progress.md` to reflect the successful debugging and passing tests. +- Consider if any updates are needed for `techContext.md` or `.clinerules`. +- Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 68e4f61..6429d65 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -10,9 +10,17 @@ - Unit tests (`bats`) for `build.sh` logic exist and pass. - Filter tests (`sh`) comparing Pandoc output against snapshots exist and pass. - E2E smoke tests (`sh`) verifying PDF generation for examples exist and pass. - - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.). -- **Docker:** Image builds, includes dependencies, uses `build.sh` as entrypoint. -- **CI:** GitHub Actions workflow lints code (`pre-commit`), builds Docker image, runs all test suites inside the container, builds example PDFs, and handles releases (uploads example PDFs, uses changelog). + - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.) and Docker build commands (`just build-docker`). +- **Docker:** + - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. + - `style.typ` is placed in `/app/lib` and included via `TYPST_FONT_PATHS`. + - Devcontainer (`.devcontainer/Dockerfile`) is Alpine-based with dev tools and Docker-in-Docker. + - Fedora-based image (`Dockerfile.fedora`) exists but is likely deprecated. +- **Testing:** + - Unit, Filter, E2E tests run inside the production container in CI. + - Docker usage tests (`tests/docker.bats`) verify container interaction (now passing after debugging). +- **CI:** GitHub Actions workflow lints code, builds the *production* Docker image, runs all test suites (unit, filter, E2E, docker), builds example PDFs using the container, pushes images, and handles releases. +- **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. ## What's Left to Build (Potential Future Work) @@ -21,24 +29,21 @@ - Enhance style customization (more parameters in `style.typ` or allow custom style files). - **Phase 3: Technical Challenges & Refinements:** - Improve/Fix stdin/stdout support for Typst format output (overrides currently ignored). - - Refactor `typst-cv.lua` and `style.typ`: - - Fix swapped `event-date`/`company-location` function usage. - - Rename functions (e.g., `date()`, `location()`). - - Add generic `{side="..."}` attribute support. - - Support multiple attributes per heading. - - Add support for divs like `#aside`. + - Refactor `typst-cv.lua` and `style.typ` (as previously noted). - Explore dynamic font installation (e.g., `fontist`). - - Separate Docker images for development (Debian/Fedora) and production (Alpine). -- **Phase 4: Rebranding & Marketing:** - - Rebrand project (name, repository, Docker Hub). - - Create GitHub Action for publishing documentation (e.g., to GitHub Pages). +- **Optimize Dockerfiles:** Look for ways to share layers/stages between `Dockerfile` and `.devcontainer/Dockerfile.ubuntu`. +- **(Potentially)** Remove `Dockerfile.fedora`. +- **Phase 4: Rebranding & Marketing:** (As previously noted) ## Current Status -- **Phase 1 Complete:** Testing framework established, examples depersonalized, CI updated, `justfile` added. Core functionality is tested. -- **Ready for Next Phase:** Project is in a stable state, ready for planning and implementation of new features or refactoring based on user priorities. +- **Phase 1 Complete:** Testing framework established, examples depersonalized. +- **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. +- **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. +- **Docker Tests Passing:** All test suites, including Docker interaction tests, are passing after significant debugging. +- **Ready for Next Phase:** Project is stable, build/test process is robust, ready for further feature development, refactoring, or optimization. ## Known Issues - **Minor Layout Quirk:** README mentions side content (`{date}`, `{location}`) might cause minor spacing variations depending on Typst layout. (Low priority Typst/styling detail). -- **Overrides Ignored for Typst Stdout:** `--set`/`TYPSTCV_*` overrides are currently ignored if outputting Typst format *to stdout* due to limitations in the pipeline. +- **Overrides Ignored for Typst Stdout:** `--set`/`TYPSTCV_*` overrides are currently ignored if outputting Typst format *to stdout* due to limitations in the pipeline. (This was not tested/fixed in this session). diff --git a/memory-bank/techContext.md b/memory-bank/techContext.md index a082a1c..9ce96cd 100644 --- a/memory-bank/techContext.md +++ b/memory-bank/techContext.md @@ -7,37 +7,57 @@ - **Lua:** Scripting language used for Pandoc filters (`linkify.lua`, `typst-cv.lua`). Filters are executed by Pandoc. - **Markdown:** Input format (CommonMark with Pandoc extensions like YAML frontmatter, attributes, classes). - **YAML:** Used within Markdown frontmatter for metadata (`author`, `title`, `links`, etc.). -- **Shell Script (Bash/sh):** Proposed for `build.sh` to replace `justfile`. Needs to be POSIX-compliant for broader compatibility (especially within minimal Docker containers like Alpine, though the current base is Fedora). -- **Docker:** Containerization platform used for packaging and distribution. `Dockerfile` defines the image build process. -- **Just:** Task runner used for local development tasks (linting, testing, building examples). Invoked via `just `. -- **Bats (Bash Automated Testing System):** Used for unit testing `build.sh`. Installed via Git submodule in `tests/bats`. Includes `bats-support` and `bats-assert` helpers. - -## Key Files & Locations (within Docker image) - -- **Pandoc Filters (`*.lua`):** Copied to `$PANDOC_DATA_DIR/filters/` (e.g., `/usr/share/pandoc-3.1.11.1/filters/`). -- **Pandoc Templates (`typst-*.typ`):** Copied to `$PANDOC_DATA_DIR/data/templates/` (e.g., `/usr/share/pandoc-3.1.11.1/data/templates/`). -- **Typst Style Package (`style.typ`, `typst.toml`):** Copied to `$TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0/` (e.g., `/usr/local/share/typst/packages/local/pandoc-cv/0.1.0/`). -- **Typst Executable:** `/usr/bin/typst`. -- **Fonts:** Installed via `dnf` into system font directories (e.g., `/usr/share/fonts/`). `TYPST_FONT_PATHS` environment variable points Typst to these. -- **Build Script (`build.sh` - Proposed):** To be copied likely to `/usr/local/bin/`. +- **Shell Script (Bash):** `build.sh` orchestrates the build process. +- **Docker:** Containerization platform. + - `Dockerfile`: Defines the primary **Alpine-based production image**. Uses multi-stage builds for fonts and Typst installation. + - `Dockerfile.fedora`: Previous Fedora-based image (potentially deprecated). + - `.devcontainer/Dockerfile`: Defines the **Alpine-based development environment** with dev tools and Docker-in-Docker. +- **Just:** Task runner for local development (linting, testing, building examples, building Docker images). Invoked via `just `. +- **Bats (Bash Automated Testing System):** Used for unit tests (`tests/unit/build_sh.bats`), filter tests (`tests/filter/filters.bats`), and Docker usage tests (`tests/docker.bats`). Installed via Git submodule in `tests/bats`. Includes `bats-support` and `bats-assert` helpers. + +## Key Files & Locations (within Production Docker image - `Dockerfile`) + +- **Pandoc Filters (`*.lua`):** Copied to `$PANDOC_DATA_DIR/filters/` (e.g., `/usr/share/pandoc/filters/`). +- **Pandoc Templates (`typst-*.typ`):** Copied to `$PANDOC_DATA_DIR/templates/` (e.g., `/usr/share/pandoc/templates/`). +- **Typst Style File (`style.typ`):** Copied to `/app/lib/style.typ`. Not treated as a package anymore. +- **Typst Executable:** `/usr/local/bin/typst` (Manually installed v0.12.0). +- **Typst Packages (`fontawesome`):** Copied from builder stage to `$TYPST_PACKAGE_PATH` (e.g., `/usr/share/typst/packages`). +- **Fonts:** Installed via `fontist` (builder stage) and `apk` into system font directories (e.g., `/usr/share/fonts/`). `TYPST_FONT_PATHS` environment variable points Typst to font directories *and* `/app/lib` (for `style.typ` import). +- **Build Script (`build.sh`):** Copied to `/usr/local/bin/build.sh`. - **Working Directory:** `/data` (users mount their project here). -## Dependencies & Versions +## Dependencies & Versions (Production Image - `Dockerfile`) -- Specific versions of Fedora, Alpine (for builder stage), Pandoc, Just, Typst, and fonts are pinned in the `Dockerfile`. This ensures reproducibility. -- External Typst packages (`@preview/fontawesome`) are also versioned. +- Base Image: Alpine 3.21. +- Builder Stages: Ruby 3.2-slim (for fontist), Alpine 3.21 (for Typst). +- Runtime Dependencies (installed via `apk`): `bash`, `pandoc`, `fontconfig`, `font-awesome`. +- Manually Installed: Typst v0.12.0 (downloaded binary). +- Typst Packages: `@preview/fontawesome` (installed via `typst update` in builder stage). +- Fonts: Installed via `fontist` (using `fontist-manifest.yml`) and `apk`. +- `git`, `wget`, `tar` are only present in builder stages. ## Build/Execution/Testing Environment -- Builds and final execution are designed to run within the provided Docker container (`kadykov/typst-cv`), which includes all necessary dependencies (Pandoc, Typst, Lua filters, Typst packages, fonts). -- Local development and testing utilize the devcontainer environment, which includes `bats-core`, `just`, `pre-commit`, and other development tools. -- The `build.sh` script relies on standard POSIX shell commands and the installed tools (Pandoc, Typst). -- Tests (`bats`, `sh`) are run from the project root. -- CI (GitHub Actions) runs linters and then executes the test suite inside the built Docker container. -- Environment variables (`TYPSTCV_*`, `PANDOC_DATA_DIR`, `TYPST_PACKAGE_PATH`, `TYPST_FONT_PATHS`) play a role in configuration, especially within Docker/CI. +- **Production Execution:** Designed to run within the built Docker container (based on `Dockerfile`), which includes runtime dependencies. User mounts their source files to `/data`. +- **Local Development:** Utilizes the **Alpine-based devcontainer** environment (`.devcontainer/Dockerfile`), which includes dev tools (`just`, `pre-commit`, `shellcheck`, etc.) and **Docker-in-Docker**. +- **Build Script (`build.sh`):** Relies on standard shell commands and tools available in the production container (Pandoc, Typst). +- **Testing:** + - Unit, Filter, E2E tests (`bats`, `sh`) are run *inside* the built production container in CI, mounting only `tests/fixtures`. + - Docker usage tests (`tests/docker.bats`) are run *on the CI host* against the built production container. + - Local testing uses `just test`. +- **CI (GitHub Actions):** + - Runs linters (`pre-commit`). + - Builds the production Docker image (`Dockerfile`). + - Runs unit, filter, E2E tests inside the container. + - Runs Docker usage tests on the host. + - Builds example PDFs using the container. + - Pushes tagged images to Docker Hub. + - Creates GitHub Releases. +- **Environment Variables:** `TYPSTCV_*` (for overrides), `PANDOC_DATA_DIR`, `TYPST_PACKAGE_PATH`, `TYPST_FONT_PATHS`, `APP_LIB_DIR` play roles in configuration within the Docker image. ## Constraints -- The system relies on specific versions of tools being available. -- Correct placement of filters, templates, and packages within the Docker image (or host system if run outside Docker) is crucial. -- Font availability is essential for correct rendering. +- Relies on specific versions of tools (Typst, fontist) pinned in `Dockerfile`. +- Correct placement of filters, templates, and `style.typ` within the Docker image is crucial. +- Font availability (via fontist and apk) is essential. +- Network access is required during Docker build to download Typst and update packages. From a1d37691a2c039f1a38a714571a2b16980bad146 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 21:59:24 +0000 Subject: [PATCH 36/63] Refactor Docker build tasks: remove Fedora image build and update comments for clarity --- justfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/justfile b/justfile index 0153489..e28ba01 100644 --- a/justfile +++ b/justfile @@ -44,10 +44,6 @@ build-examples: ./build.sh --output-dir examples tests/fixtures/example-letter.md --type letter @echo "Example PDFs built in ./examples/" -# Build the default Docker image (Alpine-based) +# Build the default Docker image build-docker tag='latest': docker build --pull -t typst-cv:{{tag}} -f Dockerfile . - -# Build the Fedora-based Docker image -build-docker-fedora tag='latest-fedora': - docker build --pull -t typst-cv:{{tag}} -f Dockerfile.fedora . From 0918f1df756a49cc6f63f3894ed0968da213b4b4 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 22:11:05 +0000 Subject: [PATCH 37/63] Refactor CI workflow and Justfile: update Docker testing steps, enhance internal test execution, and clarify comments --- .github/workflows/ci.yml | 29 ++++++++++++----------------- justfile | 5 ++++- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05997fb..88b0b6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,28 +69,23 @@ jobs: - name: Checkout code (including submodules for bats) uses: actions/checkout@v4 with: - submodules: recursive # Needed to get bats-core for docker tests + submodules: recursive # Needed for bats-core and test helpers - - name: Install Bats and File command (for Docker usage tests) - run: sudo apt-get update && sudo apt-get install -y bats file - - - name: Run tests inside the container (using production image, mounting only fixtures) + - name: Build Devcontainer Image for Testing run: | - echo "Running tests..." - # Mount only fixtures to /test-fixtures, run tests relative to /data (workdir) - # Note: Tests might need adjustment if they expect to be run from project root - # Assuming bats and sh scripts handle paths correctly when run from /data - docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - bats /test-fixtures/../unit/build_sh.bats # Adjust path relative to mount point if needed + docker build -t typst-cv-devcontainer:latest -f .devcontainer/Dockerfile.ubuntu . - docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - bats /test-fixtures/../filter/filters.bats # Adjust path relative to mount point if needed + - name: Install Bats and File command (for Docker usage tests on host) + run: sudo apt-get update && sudo apt-get install -y bats file - docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - sh /test-fixtures/../test_e2e.sh # Adjust path relative to mount point if needed - echo "Container tests finished." + - name: Run internal tests (unit, filter, e2e) inside Devcontainer Image + run: | + echo "Running internal tests..." + # Run using the devcontainer image, mounting the full workspace + docker run --rm -v "${PWD}:/workspaces/typstCV" --workdir /workspaces/typstCV typst-cv-devcontainer:latest just test-internal + echo "Internal tests finished." - - name: Run Docker usage tests (on runner host) + - name: Run Docker usage tests (on runner host against production image) run: | echo "Running Docker usage tests..." # These tests run on the host, interacting with the built testing image diff --git a/justfile b/justfile index e28ba01..46690b2 100644 --- a/justfile +++ b/justfile @@ -31,9 +31,12 @@ test-docker: {{bats_executable}} tests/docker.bats # Run all tests (including Docker tests) -test: test-unit test-filter test-e2e test-docker +test: test-internal test-docker @echo "All tests passed!" +# Run internal tests (unit, filter, e2e) - suitable for CI container execution +test-internal: test-unit test-filter test-e2e + # Clean temporary files (if any - currently handled by tests) clean: @echo "No specific clean actions defined." From 1b87fa1d7ad63be4ac7a03dda6d78e7a17aac08f Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 22:11:33 +0000 Subject: [PATCH 38/63] Fix CI workflow to execute internal tests in devcontainer and Docker usage tests on host runner --- memory-bank/activeContext.md | 56 ++++++++++++++++-------------------- memory-bank/progress.md | 11 +++---- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 25effe7..117858d 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,45 +1,37 @@ -# Active Context: Debugging Docker Tests & Switching Devcontainer (2025-04-28) +# Active Context: Fixing CI Test Execution (2025-04-29) ## Current Focus -Debugging and fixing the Docker usage tests (`tests/docker.bats`) within the new Ubuntu-based Docker-in-Docker (DinD) devcontainer environment. +Correcting the GitHub Actions CI workflow (`.github/workflows/ci.yml`) to properly execute all test suites. + +## Problem Identified + +- Local tests (`just test`) were passing in the devcontainer. +- CI tests were failing with `Error: Only one input file allowed.` during the unit/filter/E2E test execution step. +- **Root Cause:** The CI workflow was attempting to run the unit, filter, and E2E tests (which require Bats and other dev tools) *inside* the minimal production Docker image (`kadykov/typst-cv:testing`). This image lacks the necessary testing tools, and the volume mounting/working directory setup was incorrect for the test scripts' expectations. ## Recent Actions (This Session) -- **Created Production Dockerfile:** Defined `Dockerfile` (Alpine-based) for the self-contained production image, including multi-stage builds, pinned Typst v0.12.0, and necessary dependencies/assets. -- **Updated CI:** Modified `.github/workflows/ci.yml` to build and test using the new production `Dockerfile`. -- **Added Docker Tests:** Created `tests/docker.bats` to test container interaction. -- **Updated `justfile`:** Added `build-docker` and `test-docker` recipes. -- **Refactored Package Handling:** - - Reverted templates (`typst-cv.typ`, `typst-letter.typ`) to use local package import (`@local/pandoc-cv:0.1.0`). - - Updated production `Dockerfile` to copy `style.typ` and `typst.toml` into the package structure. - - Updated devcontainer `Dockerfile` (`.devcontainer/Dockerfile.ubuntu`) to create symlinks for `style.typ` and `typst.toml` into the package structure. - - Added Pandoc comment syntax (`$--`) to template files to avoid linter issues. -- **Debugged `build.sh`:** Made several adjustments to handle Pandoc `--data-dir`, `--resource-path`, stdin (`-`) argument parsing, and `typst compile` arguments for the override pipeline. -- **Identified Devcontainer Issue:** Diagnosed Docker test failures as stemming from Docker-out-of-Docker limitations with host path mounts in the Alpine devcontainer. -- **Planned Devcontainer Switch:** - - Created a new Ubuntu-based devcontainer Dockerfile (`.devcontainer/Dockerfile.ubuntu`) with necessary tools. - - Updated `.devcontainer/devcontainer.json` to use the new Dockerfile and enable the Docker-in-Docker feature. - - Simplified `tests/docker.bats` to use direct host path mounts, anticipating the DinD setup. -- **Executed Tests:** Ran `just build-docker && just test` multiple times. -- **Fixed Path Resolution:** Modified `build.sh` and `typst-cv.lua` to correctly handle image paths relative to the input file using Typst's `--root` argument, resolving E2E test failures. -- **Refactored Photo Attribute:** Changed from `{photo='image(...)'}` to `{photo="path" photowidth="..."}` for better usability, updating `typst-cv.lua` and test fixtures (`*.md`) accordingly. -- **Fixed Filter Tests:** Made `typst-cv.lua` robust against missing metadata during filter tests. -- **Fixed Docker Tests:** - - Used an image-free fixture (`example-cv-stdin.md`) for the stdin test. - - Corrected the `--set` key from `name` to `author` in the override test. - - Ensured the override test explicitly requested the correct output filename (`--output override.pdf`). - - Investigated and resolved issues preventing the output file creation in the override test. +- **Analyzed CI Failure:** Diagnosed the incorrect execution context for internal tests in the CI workflow. +- **Planned CI Fix:** Determined the need to run internal tests (unit, filter, E2E) in an environment similar to the devcontainer, separate from the Docker usage tests (`tests/docker.bats`). +- **Updated `justfile`:** + - Added a new recipe `test-internal: test-unit test-filter test-e2e` specifically for running tests that don't involve direct Docker interaction testing. + - Modified the main `test` recipe to `test: test-internal test-docker`. +- **Updated `.github/workflows/ci.yml`:** + - Added a step to build the devcontainer image (`typst-cv-devcontainer:latest`) from `.devcontainer/Dockerfile.ubuntu`. + - Replaced the failing internal test execution step to use the `typst-cv-devcontainer:latest` image, mount the full workspace (`${PWD}:/workspaces/typstCV`), set the working directory (`/workspaces/typstCV`), and run `just test-internal`. + - Kept the "Run Docker usage tests (on runner host against production image)" step unchanged, as it correctly tests the production image's external interface. ## Decisions & Notes -- Devcontainer successfully switched to Ubuntu (DinD), simplifying Docker testing. -- Photo handling now uses separate `photo` (path) and optional `photowidth` attributes in Markdown for improved user experience. -- Path resolution for images relies on Typst's `--root` argument, set correctly by `build.sh`. -- All test suites (unit, filter, E2E, docker) are now passing. +- Internal tests (unit, filter, E2E) require the development environment context (tools like Bats). +- Docker usage tests (`tests/docker.bats`) specifically test the *production* image interface from the outside and should run on the host runner. +- Separating the execution context for these two types of tests in CI is crucial. +- Using the devcontainer Dockerfile (`.devcontainer/Dockerfile.ubuntu`) provides the necessary environment for internal tests in CI. ## Immediate Next Steps -- Update `progress.md` to reflect the successful debugging and passing tests. +- **User Action:** Commit and push the changes to `justfile` and `.github/workflows/ci.yml` to trigger the CI workflow and verify the fix. +- **Update `progress.md`:** Reflect the CI fix and current project status. - Consider if any updates are needed for `techContext.md` or `.clinerules`. -- Complete the task. +- Complete the task once CI verification is successful. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 6429d65..4d01006 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -17,9 +17,9 @@ - Devcontainer (`.devcontainer/Dockerfile`) is Alpine-based with dev tools and Docker-in-Docker. - Fedora-based image (`Dockerfile.fedora`) exists but is likely deprecated. - **Testing:** - - Unit, Filter, E2E tests run inside the production container in CI. - - Docker usage tests (`tests/docker.bats`) verify container interaction (now passing after debugging). -- **CI:** GitHub Actions workflow lints code, builds the *production* Docker image, runs all test suites (unit, filter, E2E, docker), builds example PDFs using the container, pushes images, and handles releases. + - Unit, Filter, E2E tests (`just test-internal`) run inside a devcontainer-like image in CI. + - Docker usage tests (`tests/docker.bats`, run via `just test-docker`) verify production container interaction on the CI host runner. +- **CI:** GitHub Actions workflow lints code, builds the *production* Docker image (`Dockerfile`), builds the *devcontainer* image (`.devcontainer/Dockerfile.ubuntu`), runs internal tests in the devcontainer image, runs Docker usage tests against the production image, builds example PDFs using the production container, pushes images, and handles releases. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. ## What's Left to Build (Potential Future Work) @@ -40,8 +40,9 @@ - **Phase 1 Complete:** Testing framework established, examples depersonalized. - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. -- **Docker Tests Passing:** All test suites, including Docker interaction tests, are passing after significant debugging. -- **Ready for Next Phase:** Project is stable, build/test process is robust, ready for further feature development, refactoring, or optimization. +- **Docker Tests Passing:** All test suites, including Docker interaction tests, are passing locally after significant debugging. +- **CI Workflow Corrected:** CI pipeline now uses appropriate environments for internal tests (devcontainer image) and Docker usage tests (host runner + production image). Awaiting verification via CI run. +- **Ready for Next Phase:** Project is stable, build/test process is robust (pending CI verification), ready for further feature development, refactoring, or optimization. ## Known Issues From 29e7eea0c8eef71de80aa4c6853f35d5e4a02d79 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 22:32:38 +0000 Subject: [PATCH 39/63] Add diagnostic step to verify submodule checkout in CI workflow --- .github/workflows/ci.yml | 7 +++++++ memory-bank/activeContext.md | 39 +++++++++++++++++------------------- memory-bank/progress.md | 6 +++--- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88b0b6c..3d40c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,13 @@ jobs: with: submodules: recursive # Needed for bats-core and test helpers + - name: Verify Submodule Checkout + run: | + echo "Listing tests/bats/bin on runner:" + ls -l tests/bats/bin || echo "!!! tests/bats/bin not found on runner" + echo "Listing tests/test_helper on runner:" + ls -l tests/test_helper || echo "!!! tests/test_helper not found on runner" + - name: Build Devcontainer Image for Testing run: | docker build -t typst-cv-devcontainer:latest -f .devcontainer/Dockerfile.ubuntu . diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 117858d..731c037 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -2,36 +2,33 @@ ## Current Focus -Correcting the GitHub Actions CI workflow (`.github/workflows/ci.yml`) to properly execute all test suites. +Diagnosing and correcting the GitHub Actions CI workflow (`.github/workflows/ci.yml`) to properly execute all test suites. ## Problem Identified -- Local tests (`just test`) were passing in the devcontainer. -- CI tests were failing with `Error: Only one input file allowed.` during the unit/filter/E2E test execution step. -- **Root Cause:** The CI workflow was attempting to run the unit, filter, and E2E tests (which require Bats and other dev tools) *inside* the minimal production Docker image (`kadykov/typst-cv:testing`). This image lacks the necessary testing tools, and the volume mounting/working directory setup was incorrect for the test scripts' expectations. +- Local tests (`just test`) pass in the devcontainer. +- CI internal tests (`just test-internal` running in devcontainer image) failed with `sh: 1: tests/bats/bin/bats: not found`. +- **Hypothesis:** The `actions/checkout@v4` step might not be checking out the Git submodules (`tests/bats`, `tests/test_helper`) correctly on the CI runner, despite `submodules: recursive` being set. ## Recent Actions (This Session) -- **Analyzed CI Failure:** Diagnosed the incorrect execution context for internal tests in the CI workflow. -- **Planned CI Fix:** Determined the need to run internal tests (unit, filter, E2E) in an environment similar to the devcontainer, separate from the Docker usage tests (`tests/docker.bats`). -- **Updated `justfile`:** - - Added a new recipe `test-internal: test-unit test-filter test-e2e` specifically for running tests that don't involve direct Docker interaction testing. - - Modified the main `test` recipe to `test: test-internal test-docker`. -- **Updated `.github/workflows/ci.yml`:** - - Added a step to build the devcontainer image (`typst-cv-devcontainer:latest`) from `.devcontainer/Dockerfile.ubuntu`. - - Replaced the failing internal test execution step to use the `typst-cv-devcontainer:latest` image, mount the full workspace (`${PWD}:/workspaces/typstCV`), set the working directory (`/workspaces/typstCV`), and run `just test-internal`. - - Kept the "Run Docker usage tests (on runner host against production image)" step unchanged, as it correctly tests the production image's external interface. +- **Analyzed Initial CI Failure:** Diagnosed the incorrect execution context (running tests in production image) for internal tests. +- **Planned & Implemented CI Fix v1:** + - Added `test-internal` recipe to `justfile`. + - Modified `.github/workflows/ci.yml` to build the devcontainer image and run `just test-internal` within it, mounting the workspace. +- **Analyzed Second CI Failure (`bats: not found`):** Identified the failure point as the inability to find the Bats executable within the devcontainer test run. +- **Added Diagnostic Step to CI:** Modified `.github/workflows/ci.yml` to add a "Verify Submodule Checkout" step immediately after `actions/checkout@v4`. This step uses `ls -l` to check if the submodule directories (`tests/bats/bin`, `tests/test_helper`) are populated on the CI runner host *before* the Docker steps run. ## Decisions & Notes -- Internal tests (unit, filter, E2E) require the development environment context (tools like Bats). -- Docker usage tests (`tests/docker.bats`) specifically test the *production* image interface from the outside and should run on the host runner. -- Separating the execution context for these two types of tests in CI is crucial. -- Using the devcontainer Dockerfile (`.devcontainer/Dockerfile.ubuntu`) provides the necessary environment for internal tests in CI. +- The `bats: not found` error points to an issue with the availability of the submodule files within the CI job's execution environment. +- Verifying the checkout step directly on the runner is the next logical diagnostic step. +- If submodules *are* checked out correctly on the runner, the issue might be with the Docker volume mount (`-v "${PWD}:/workspaces/typstCV"`) or permissions within the container. +- If submodules *are not* checked out correctly, the `actions/checkout@v4` step or its configuration needs further investigation. ## Immediate Next Steps -- **User Action:** Commit and push the changes to `justfile` and `.github/workflows/ci.yml` to trigger the CI workflow and verify the fix. -- **Update `progress.md`:** Reflect the CI fix and current project status. -- Consider if any updates are needed for `techContext.md` or `.clinerules`. -- Complete the task once CI verification is successful. +- **User Action:** Commit and push the latest change to `.github/workflows/ci.yml` (adding the verification step) to trigger the CI workflow. +- **Analyze CI Output:** Examine the output of the "Verify Submodule Checkout" step in the new CI run. +- **Update `progress.md`:** Reflect the ongoing CI debugging status. +- Based on the CI output, determine the next fix (e.g., adjust checkout, adjust Docker mount, add explicit submodule commands). diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 4d01006..1837666 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -40,9 +40,9 @@ - **Phase 1 Complete:** Testing framework established, examples depersonalized. - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. -- **Docker Tests Passing:** All test suites, including Docker interaction tests, are passing locally after significant debugging. -- **CI Workflow Corrected:** CI pipeline now uses appropriate environments for internal tests (devcontainer image) and Docker usage tests (host runner + production image). Awaiting verification via CI run. -- **Ready for Next Phase:** Project is stable, build/test process is robust (pending CI verification), ready for further feature development, refactoring, or optimization. +- **Docker Tests Passing:** All test suites pass locally. +- **CI Workflow Debugging:** CI pipeline updated to use devcontainer image for internal tests, but failed (`bats: not found`). Added diagnostic step to verify submodule checkout via `actions/checkout@v4`. Awaiting results from next CI run. +- **Ready for Next Phase:** Blocked on resolving CI test failures. ## Known Issues From 0e50e5fec85b16a80fe39d08ad37b6a0aa895172 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 22:48:23 +0000 Subject: [PATCH 40/63] Fix CI workflow to ensure submodules are initialized and updated, and resolve linter errors in the release job --- .github/workflows/ci.yml | 16 +++++++--------- memory-bank/activeContext.md | 25 ++++++++++++++----------- memory-bank/progress.md | 4 ++-- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d40c9d..feaf696 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,17 +66,14 @@ jobs: cache-from: type=registry,ref=${{ env.IMAGE }}:testing # Cache from previous testing tag if exists cache-to: type=inline - - name: Checkout code (including submodules for bats) + - name: Checkout code uses: actions/checkout@v4 + # Keep submodules: recursive just in case, but rely on explicit update with: - submodules: recursive # Needed for bats-core and test helpers + submodules: recursive - - name: Verify Submodule Checkout - run: | - echo "Listing tests/bats/bin on runner:" - ls -l tests/bats/bin || echo "!!! tests/bats/bin not found on runner" - echo "Listing tests/test_helper on runner:" - ls -l tests/test_helper || echo "!!! tests/test_helper not found on runner" + - name: Initialize and Update Submodules + run: git submodule update --init --recursive - name: Build Devcontainer Image for Testing run: | @@ -182,6 +179,7 @@ jobs: - name: Create release on GitHub uses: ncipollo/release-action@v1 with: + # Ensure no duplicate keys exist below tag: ${{ steps.changelog_reader.outputs.version }} name: Release ${{ steps.changelog_reader.outputs.version }} body: ${{ steps.changelog_reader.outputs.changes }} @@ -189,4 +187,4 @@ jobs: draft: ${{ steps.changelog_reader.outputs.status == 'unreleased' }} allowUpdates: true token: ${{ secrets.GITHUB_TOKEN }} - artifacts: ./release-assets/*.pdf # Upload PDFs from the download directory + artifacts: ./release-assets/*.pdf diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 731c037..0e60bb2 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -8,7 +8,8 @@ Diagnosing and correcting the GitHub Actions CI workflow (`.github/workflows/ci. - Local tests (`just test`) pass in the devcontainer. - CI internal tests (`just test-internal` running in devcontainer image) failed with `sh: 1: tests/bats/bin/bats: not found`. -- **Hypothesis:** The `actions/checkout@v4` step might not be checking out the Git submodules (`tests/bats`, `tests/test_helper`) correctly on the CI runner, despite `submodules: recursive` being set. +- **Root Cause Confirmed:** Diagnostic step showed `actions/checkout@v4` was not checking out submodules on the CI runner. +- **Secondary Issue:** Linter errors detected duplicate keys in the `release` job after a previous file modification. ## Recent Actions (This Session) @@ -16,19 +17,21 @@ Diagnosing and correcting the GitHub Actions CI workflow (`.github/workflows/ci. - **Planned & Implemented CI Fix v1:** - Added `test-internal` recipe to `justfile`. - Modified `.github/workflows/ci.yml` to build the devcontainer image and run `just test-internal` within it, mounting the workspace. -- **Analyzed Second CI Failure (`bats: not found`):** Identified the failure point as the inability to find the Bats executable within the devcontainer test run. -- **Added Diagnostic Step to CI:** Modified `.github/workflows/ci.yml` to add a "Verify Submodule Checkout" step immediately after `actions/checkout@v4`. This step uses `ls -l` to check if the submodule directories (`tests/bats/bin`, `tests/test_helper`) are populated on the CI runner host *before* the Docker steps run. +- **Analyzed Second CI Failure (`bats: not found`):** Identified the failure point as the inability to find the Bats executable. +- **Added Diagnostic Step to CI:** Added a verification step using `ls` which confirmed submodules were not checked out by `actions/checkout@v4`. +- **Implemented CI Fix v2 (Submodules):** + - Removed the diagnostic `ls` step. + - Added an explicit `git submodule update --init --recursive` command in `.github/workflows/ci.yml` immediately after the `actions/checkout@v4` step to ensure submodules are initialized and updated. +- **Fixed Linter Errors:** Corrected duplicate keys (`allowUpdates`, `token`, `artifacts`) in the `release` job within `.github/workflows/ci.yml` by rewriting the affected step. ## Decisions & Notes -- The `bats: not found` error points to an issue with the availability of the submodule files within the CI job's execution environment. -- Verifying the checkout step directly on the runner is the next logical diagnostic step. -- If submodules *are* checked out correctly on the runner, the issue might be with the Docker volume mount (`-v "${PWD}:/workspaces/typstCV"`) or permissions within the container. -- If submodules *are not* checked out correctly, the `actions/checkout@v4` step or its configuration needs further investigation. +- Explicitly running `git submodule update --init --recursive` is the most reliable way to ensure submodules are available in the CI environment. +- Rewriting problematic YAML sections can resolve hidden duplication issues sometimes missed by simple text diffs. ## Immediate Next Steps -- **User Action:** Commit and push the latest change to `.github/workflows/ci.yml` (adding the verification step) to trigger the CI workflow. -- **Analyze CI Output:** Examine the output of the "Verify Submodule Checkout" step in the new CI run. -- **Update `progress.md`:** Reflect the ongoing CI debugging status. -- Based on the CI output, determine the next fix (e.g., adjust checkout, adjust Docker mount, add explicit submodule commands). +- **User Action:** Commit and push the latest changes to `.github/workflows/ci.yml` to trigger the CI workflow. +- **Analyze CI Output:** Verify that the `git submodule update` step runs successfully and that the subsequent `just test-internal` step passes (specifically, that `bats` is now found). +- **Update `progress.md`:** Reflect the latest CI fix attempt. +- Complete the task once CI verification is successful. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 1837666..3c62859 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -41,8 +41,8 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Docker Tests Passing:** All test suites pass locally. -- **CI Workflow Debugging:** CI pipeline updated to use devcontainer image for internal tests, but failed (`bats: not found`). Added diagnostic step to verify submodule checkout via `actions/checkout@v4`. Awaiting results from next CI run. -- **Ready for Next Phase:** Blocked on resolving CI test failures. +- **CI Workflow Fix Attempted:** Added explicit `git submodule update --init --recursive` step to CI workflow to address `bats: not found` error. Also fixed linter errors in the `release` job. Awaiting results from next CI run. +- **Ready for Next Phase:** Blocked on resolving CI test failures (pending verification of latest fix). ## Known Issues From 07ef11db551ac2b64dd06d25f9b72f0049bcfbaf Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Tue, 29 Apr 2025 23:26:35 +0000 Subject: [PATCH 41/63] Enhance CI workflow: add verification step for submodule checkout and update submodule URLs to latest versions --- .github/workflows/ci.yml | 10 ++++++++++ .gitmodules | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index feaf696..e206393 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,16 @@ jobs: - name: Initialize and Update Submodules run: git submodule update --init --recursive + # Verify submodule checkout after update + - name: Verify Submodule Checkout (Post-Update) + run: | + echo "--- Verifying Submodule Checkout ---" + echo "Listing tests/bats/bin:" + ls -l tests/bats/bin || echo "!!! tests/bats/bin NOT FOUND" + echo "Listing tests/test_helper:" + ls -l tests/test_helper || echo "!!! tests/test_helper NOT FOUND" + echo "--- Verification Done ---" + - name: Build Devcontainer Image for Testing run: | docker build -t typst-cv-devcontainer:latest -f .devcontainer/Dockerfile.ubuntu . diff --git a/.gitmodules b/.gitmodules index 28655fa..fe9d335 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "tests/bats"] path = tests/bats - url = https://github.com/bats-core/bats-core.git@v1.11.1 + url = https://github.com/bats-core/bats-core.git [submodule "tests/test_helper/bats-support"] path = tests/test_helper/bats-support - url = https://github.com/bats-core/bats-support.git@v0.3.0 + url = https://github.com/bats-core/bats-support.git [submodule "tests/test_helper/bats-assert"] path = tests/test_helper/bats-assert - url = https://github.com/bats-core/bats-assert.git@v2.1.0 + url = https://github.com/bats-core/bats-assert.git From 2eb782ea025cc01c214a9df1dc17d114f6fa073a Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 10:05:01 +0000 Subject: [PATCH 42/63] Update submodule URLs to use SSH and add submodule commits for bats and its helpers --- .gitmodules | 6 +++--- tests/bats | 1 + tests/test_helper/bats-assert | 1 + tests/test_helper/bats-support | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) create mode 160000 tests/bats create mode 160000 tests/test_helper/bats-assert create mode 160000 tests/test_helper/bats-support diff --git a/.gitmodules b/.gitmodules index fe9d335..6c47622 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,9 @@ [submodule "tests/bats"] path = tests/bats - url = https://github.com/bats-core/bats-core.git + url = git@github.com:bats-core/bats-core.git [submodule "tests/test_helper/bats-support"] path = tests/test_helper/bats-support - url = https://github.com/bats-core/bats-support.git + url = git@github.com:bats-core/bats-support.git [submodule "tests/test_helper/bats-assert"] path = tests/test_helper/bats-assert - url = https://github.com/bats-core/bats-assert.git + url = git@github.com:bats-core/bats-assert.git diff --git a/tests/bats b/tests/bats new file mode 160000 index 0000000..3172a45 --- /dev/null +++ b/tests/bats @@ -0,0 +1 @@ +Subproject commit 3172a45e55a58bbaf952e971a4b7347483842ba7 diff --git a/tests/test_helper/bats-assert b/tests/test_helper/bats-assert new file mode 160000 index 0000000..b93143a --- /dev/null +++ b/tests/test_helper/bats-assert @@ -0,0 +1 @@ +Subproject commit b93143a1bfbde41d9b7343aab0d36f3ef6549e6b diff --git a/tests/test_helper/bats-support b/tests/test_helper/bats-support new file mode 160000 index 0000000..d007fc1 --- /dev/null +++ b/tests/test_helper/bats-support @@ -0,0 +1 @@ +Subproject commit d007fc1f451abbad55204fa9c9eb3e6ed5dc5f61 From 6587839cb7bf4a53f6c116d4e437e6971faa08ca Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 12:01:04 +0000 Subject: [PATCH 43/63] Refactor Bats integration: remove submodule references and update to system-installed Bats --- .devcontainer/Dockerfile.ubuntu | 3 +++ .gitmodules | 9 --------- justfile | 2 +- tests/bats | 1 - tests/docker.bats | 5 +++-- tests/filter/filters.bats | 6 +++--- tests/test_helper/bats-assert | 1 - tests/test_helper/bats-support | 1 - tests/unit/build_sh.bats | 6 +++--- 9 files changed, 13 insertions(+), 21 deletions(-) delete mode 160000 tests/bats delete mode 160000 tests/test_helper/bats-assert delete mode 160000 tests/test_helper/bats-support diff --git a/.devcontainer/Dockerfile.ubuntu b/.devcontainer/Dockerfile.ubuntu index b514996..1ce582a 100644 --- a/.devcontainer/Dockerfile.ubuntu +++ b/.devcontainer/Dockerfile.ubuntu @@ -27,6 +27,9 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ ca-certificates \ unzip \ just \ + bats \ + bats-assert \ + bats-support \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/.gitmodules b/.gitmodules index 6c47622..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +0,0 @@ -[submodule "tests/bats"] - path = tests/bats - url = git@github.com:bats-core/bats-core.git -[submodule "tests/test_helper/bats-support"] - path = tests/test_helper/bats-support - url = git@github.com:bats-core/bats-support.git -[submodule "tests/test_helper/bats-assert"] - path = tests/test_helper/bats-assert - url = git@github.com:bats-core/bats-assert.git diff --git a/justfile b/justfile index 46690b2..c430758 100644 --- a/justfile +++ b/justfile @@ -1,7 +1,7 @@ # Justfile for common development tasks # Variables -bats_executable := "tests/bats/bin/bats" +bats_executable := "bats" # Use system-installed bats unit_tests_file := "tests/unit/build_sh.bats" filter_tests_file := "tests/filter/filters.bats" # Updated path e2e_tests_script := "tests/test_e2e.sh" diff --git a/tests/bats b/tests/bats deleted file mode 160000 index 3172a45..0000000 --- a/tests/bats +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3172a45e55a58bbaf952e971a4b7347483842ba7 diff --git a/tests/docker.bats b/tests/docker.bats index 8d9699d..36b7dd5 100755 --- a/tests/docker.bats +++ b/tests/docker.bats @@ -1,7 +1,8 @@ #!/usr/bin/env bats -load 'test_helper/bats-support/load' -load 'test_helper/bats-assert/load' +# Load Bats helpers using bats_load_library for system packages +bats_load_library 'bats-support' +bats_load_library 'bats-assert' # --- Variables --- # Assumes the image is built with the 'latest' tag locally diff --git a/tests/filter/filters.bats b/tests/filter/filters.bats index 80b3bc2..a5c6221 100755 --- a/tests/filter/filters.bats +++ b/tests/filter/filters.bats @@ -1,8 +1,8 @@ #!/usr/bin/env bats -# Load Bats helpers from submodules -load '../test_helper/bats-support/load.bash' -load '../test_helper/bats-assert/load.bash' +# Load Bats helpers using bats_load_library for system packages +bats_load_library 'bats-support' +bats_load_library 'bats-assert' # --- Configuration --- FIXTURES_DIR="./tests/fixtures" diff --git a/tests/test_helper/bats-assert b/tests/test_helper/bats-assert deleted file mode 160000 index b93143a..0000000 --- a/tests/test_helper/bats-assert +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b93143a1bfbde41d9b7343aab0d36f3ef6549e6b diff --git a/tests/test_helper/bats-support b/tests/test_helper/bats-support deleted file mode 160000 index d007fc1..0000000 --- a/tests/test_helper/bats-support +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d007fc1f451abbad55204fa9c9eb3e6ed5dc5f61 diff --git a/tests/unit/build_sh.bats b/tests/unit/build_sh.bats index d9126b6..3bfb704 100755 --- a/tests/unit/build_sh.bats +++ b/tests/unit/build_sh.bats @@ -1,8 +1,8 @@ #!/usr/bin/env bats -# Load Bats helpers from submodules -load '../test_helper/bats-support/load.bash' -load '../test_helper/bats-assert/load.bash' +# Load Bats helpers using bats_load_library for system packages +bats_load_library 'bats-support' +bats_load_library 'bats-assert' setup() { # Create a dummy input file for tests that need one From 0780bf03d3a635cad7b3a2cec61e9297154e4f3a Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 12:02:22 +0000 Subject: [PATCH 44/63] Add .dockerignore file to exclude unnecessary files from Docker image --- .dockerignore | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ebba0a2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# Git related +.git +.gitignore +.gitmodules + +# Development and CI/CD +.devcontainer +.github +.vscode +.pre-commit-config.yaml +.yamlfmt.yml +justfile + +# Tests (not needed in production image) +tests/ + +# Examples (built separately if needed, not part of base image) +examples/ + +# Documentation / Metadata +CHANGELOG.md +README.md +LICENSE +memory-bank/ + +# Temporary / Output files +*.pdf +temp.* + +# Environment files (keep example) +.env From 36c0c2dfba18c0a9284a5bf512660e4fbc0d12f8 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 12:02:47 +0000 Subject: [PATCH 45/63] Remove submodule initialization and verification steps from CI workflow as submodules are no longer used for testing dependencies --- .github/workflows/ci.yml | 18 ++----------- memory-bank/activeContext.md | 52 ++++++++++++++++++++---------------- memory-bank/progress.md | 26 +++++++++--------- 3 files changed, 43 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e206393..8dc7d88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,22 +68,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - # Keep submodules: recursive just in case, but rely on explicit update - with: - submodules: recursive - - - name: Initialize and Update Submodules - run: git submodule update --init --recursive - - # Verify submodule checkout after update - - name: Verify Submodule Checkout (Post-Update) - run: | - echo "--- Verifying Submodule Checkout ---" - echo "Listing tests/bats/bin:" - ls -l tests/bats/bin || echo "!!! tests/bats/bin NOT FOUND" - echo "Listing tests/test_helper:" - ls -l tests/test_helper || echo "!!! tests/test_helper NOT FOUND" - echo "--- Verification Done ---" + # Submodules are no longer used for testing dependencies + # submodules: recursive # Keep commented out as reference if needed - name: Build Devcontainer Image for Testing run: | diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 0e60bb2..e39a1f3 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,37 +1,43 @@ -# Active Context: Fixing CI Test Execution (2025-04-29) +# Active Context: Switched Test Dependencies from Submodules to System Packages (2025-04-30) ## Current Focus -Diagnosing and correcting the GitHub Actions CI workflow (`.github/workflows/ci.yml`) to properly execute all test suites. +Resolving CI failures related to test execution and dependency management. -## Problem Identified +## Problem Identified & Resolved -- Local tests (`just test`) pass in the devcontainer. -- CI internal tests (`just test-internal` running in devcontainer image) failed with `sh: 1: tests/bats/bin/bats: not found`. -- **Root Cause Confirmed:** Diagnostic step showed `actions/checkout@v4` was not checking out submodules on the CI runner. -- **Secondary Issue:** Linter errors detected duplicate keys in the `release` job after a previous file modification. +1. **Initial Problem:** CI tests failed (`bats: not found`) because `actions/checkout@v4` wasn't initializing submodules. +2. **Intermediate Fix:** Added explicit `git submodule update --init --recursive` to CI. +3. **New Problem:** The `docker build` step for the *production* image failed with `git@github.com: Permission denied (publickey)`. This occurred because the build context included submodule definitions using SSH URLs, but the build environment lacked SSH keys. +4. **Root Cause:** The production Docker image build context included `.git` and `tests/` directories. The testing environment (devcontainer) relied on submodules. +5. **Solution Implemented:** + * Switched test dependencies (Bats, Bats-Support, Bats-Assert) from Git submodules to system packages (`apt install`) within the devcontainer (`.devcontainer/Dockerfile.ubuntu`) and for host-run tests (`tests/docker.bats` via CI step). + * Created `.dockerignore` to exclude `.git`, `tests/`, and other non-essential files from the production Docker build context, preventing the SSH key error. + * Updated Bats test files (`tests/unit/build_sh.bats`, `tests/filter/filters.bats`) to load helpers using `bats_load_library` instead of relative submodule paths. + * Updated `justfile` to use the system `bats` executable instead of the submodule path. + * Removed the explicit `git submodule update` step from the CI workflow (`.github/workflows/ci.yml`). ## Recent Actions (This Session) -- **Analyzed Initial CI Failure:** Diagnosed the incorrect execution context (running tests in production image) for internal tests. -- **Planned & Implemented CI Fix v1:** - - Added `test-internal` recipe to `justfile`. - - Modified `.github/workflows/ci.yml` to build the devcontainer image and run `just test-internal` within it, mounting the workspace. -- **Analyzed Second CI Failure (`bats: not found`):** Identified the failure point as the inability to find the Bats executable. -- **Added Diagnostic Step to CI:** Added a verification step using `ls` which confirmed submodules were not checked out by `actions/checkout@v4`. -- **Implemented CI Fix v2 (Submodules):** - - Removed the diagnostic `ls` step. - - Added an explicit `git submodule update --init --recursive` command in `.github/workflows/ci.yml` immediately after the `actions/checkout@v4` step to ensure submodules are initialized and updated. -- **Fixed Linter Errors:** Corrected duplicate keys (`allowUpdates`, `token`, `artifacts`) in the `release` job within `.github/workflows/ci.yml` by rewriting the affected step. +- Diagnosed the SSH key failure during `docker build`. +- Discussed options (HTTPS URLs, SSH keys, system packages). +- Decided to switch to system packages for Bats and exclude test/git files from production build context. +- Created `.dockerignore`. +- Removed submodule update step from `.github/workflows/ci.yml`. +- Updated `load` commands in `.bats` files to `bats_load_library`. +- Updated `justfile` to use system `bats`. +- Verified the fix for loading helpers with the user. ## Decisions & Notes -- Explicitly running `git submodule update --init --recursive` is the most reliable way to ensure submodules are available in the CI environment. -- Rewriting problematic YAML sections can resolve hidden duplication issues sometimes missed by simple text diffs. +- Using system packages for test dependencies simplifies the CI build process and avoids SSH key issues during production image builds. +- `.dockerignore` is crucial for keeping production build contexts clean and secure. +- `bats_load_library` is the correct way to load system-installed Bats helpers. ## Immediate Next Steps -- **User Action:** Commit and push the latest changes to `.github/workflows/ci.yml` to trigger the CI workflow. -- **Analyze CI Output:** Verify that the `git submodule update` step runs successfully and that the subsequent `just test-internal` step passes (specifically, that `bats` is now found). -- **Update `progress.md`:** Reflect the latest CI fix attempt. -- Complete the task once CI verification is successful. +- Update `progress.md`. +- **User Action:** Commit the changes (`.dockerignore`, `.github/workflows/ci.yml`, `justfile`, `tests/**/*.bats`). +- **User Action:** Trigger the CI workflow and verify all tests pass. +- **User Action (Optional):** Clean up Git submodule configuration (`.gitmodules`, `git rm --cached ...`, `rm -rf ...`). +- Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 3c62859..4a8f6fe 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -1,4 +1,4 @@ -# Progress: Typst CV/Letter Generator (As of 2025-04-26) +# Progress: Typst CV/Letter Generator (As of 2025-04-30) ## What Works (Current State) @@ -8,19 +8,16 @@ - **Examples:** Generic, depersonalized example files (`example-cv.md`, `example-letter.md`) and placeholder image exist in `tests/fixtures/`. - **Testing:** - Unit tests (`bats`) for `build.sh` logic exist and pass. - - Filter tests (`sh`) comparing Pandoc output against snapshots exist and pass. + - Filter tests (`bats`) comparing Pandoc output against snapshots exist and pass. - E2E smoke tests (`sh`) verifying PDF generation for examples exist and pass. - - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.) and Docker build commands (`just build-docker`). + - Docker usage tests (`bats`) verify production container interaction. + - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.) using system-installed `bats`. - **Docker:** - - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. - - `style.typ` is placed in `/app/lib` and included via `TYPST_FONT_PATHS`. - - Devcontainer (`.devcontainer/Dockerfile`) is Alpine-based with dev tools and Docker-in-Docker. - - Fedora-based image (`Dockerfile.fedora`) exists but is likely deprecated. -- **Testing:** - - Unit, Filter, E2E tests (`just test-internal`) run inside a devcontainer-like image in CI. - - Docker usage tests (`tests/docker.bats`, run via `just test-docker`) verify production container interaction on the CI host runner. -- **CI:** GitHub Actions workflow lints code, builds the *production* Docker image (`Dockerfile`), builds the *devcontainer* image (`.devcontainer/Dockerfile.ubuntu`), runs internal tests in the devcontainer image, runs Docker usage tests against the production image, builds example PDFs using the production container, pushes images, and handles releases. + - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. + - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, and system-installed Bats for testing. +- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image, runs Docker usage tests against the production image, builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. +- **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. ## What's Left to Build (Potential Future Work) @@ -40,9 +37,10 @@ - **Phase 1 Complete:** Testing framework established, examples depersonalized. - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. -- **Docker Tests Passing:** All test suites pass locally. -- **CI Workflow Fix Attempted:** Added explicit `git submodule update --init --recursive` step to CI workflow to address `bats: not found` error. Also fixed linter errors in the `release` job. Awaiting results from next CI run. -- **Ready for Next Phase:** Blocked on resolving CI test failures (pending verification of latest fix). +- **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. +- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. +- **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). +- **Ready for Next Phase:** Project is stable, CI should pass. ## Known Issues From 2f4088c10901c241f36a91e490ecac914aef8f1a Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 14:24:02 +0000 Subject: [PATCH 46/63] Clarify comment regarding build.sh permissions in Bats test --- tests/unit/build_sh.bats | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/build_sh.bats b/tests/unit/build_sh.bats index 3bfb704..8f13fd9 100755 --- a/tests/unit/build_sh.bats +++ b/tests/unit/build_sh.bats @@ -17,8 +17,7 @@ EOF PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" SCRIPT="$PROJECT_ROOT/build.sh" - # Make build.sh executable if it's not already - chmod +x "$SCRIPT" + # build.sh should have execute permissions set by git } teardown() { From 9ea23fd125d9cf82d1222549b018f8503db221fb Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 14:35:38 +0000 Subject: [PATCH 47/63] Refactor Typst package setup: move symlink creation from Dockerfile to postCreateCommand in devcontainer.json --- .devcontainer/Dockerfile.ubuntu | 22 +++++++++++----------- .devcontainer/devcontainer.json | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.devcontainer/Dockerfile.ubuntu b/.devcontainer/Dockerfile.ubuntu index 1ce582a..0018bbe 100644 --- a/.devcontainer/Dockerfile.ubuntu +++ b/.devcontainer/Dockerfile.ubuntu @@ -62,6 +62,17 @@ RUN gem install fontist --no-document --version "~> 1.21.1" # Switch back to vscode user provided by base image USER vscode +# --- Setup Local Typst Package Symlink for Development/Testing --- +# Define the target package path using the standard ENV var name +# Use the user's local share directory now +ENV TYPST_PACKAGE_PATH=/home/vscode/.local/share/typst/packages +ENV DEV_PKG_DIR=$TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0 +# Create the directory structure and symlinks during build +# These point to the workspace location, which will be mounted at runtime +RUN mkdir -p $DEV_PKG_DIR && \ + ln -sf /workspaces/typstCV/style.typ $DEV_PKG_DIR/style.typ && \ + ln -sf /workspaces/typstCV/typst.toml $DEV_PKG_DIR/typst.toml + # Install fonts using fontist (run as vscode user) COPY --chown=vscode:vscode fontist-manifest.yml /tmp/fontist-manifest.yml RUN fontist update --quiet \ @@ -70,17 +81,6 @@ RUN fontist update --quiet \ && rm /tmp/fontist-manifest.yml \ && sudo fc-cache -fv # Update system font cache -# --- Setup Local Typst Package Symlink for Development --- -# Define the target package path using the standard ENV var name -# Use the user's local share directory now -ENV TYPST_PACKAGE_PATH=/home/vscode/.local/share/typst/packages -ENV DEV_PKG_DIR=$TYPST_PACKAGE_PATH/local/pandoc-cv/0.1.0 -# Create the directory structure -RUN mkdir -p $DEV_PKG_DIR -# Create symlinks from the workspace files to the package directory -# This runs post-checkout, so workspace files exist -# Use a postCreateCommand in devcontainer.json instead of RUN here - # Set Font Path (includes fontist user dir and system FA6 dir) ENV TYPST_FONT_PATHS=/home/vscode/.fontist/fonts:/usr/local/share/fonts/fontawesome6 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b82f328..932b76b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,7 +17,7 @@ "moby": true } }, - "postCreateCommand": "mkdir -p /home/vscode/.local/share/typst/packages/local/pandoc-cv/0.1.0 && ln -sf /workspaces/typstCV/style.typ /home/vscode/.local/share/typst/packages/local/pandoc-cv/0.1.0/style.typ && ln -sf /workspaces/typstCV/typst.toml /home/vscode/.local/share/typst/packages/local/pandoc-cv/0.1.0/typst.toml && pre-commit install", + "postCreateCommand": "pre-commit install", "remoteUser": "vscode", "customizations": { "vscode": { From b430cbd725fb052dd1fde5bdf279dd60fa966d14 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 14:37:05 +0000 Subject: [PATCH 48/63] Update CI workflow to install Bats helpers and modify active context documentation for resolved issues --- .github/workflows/ci.yml | 4 ++-- memory-bank/activeContext.md | 30 ++++++++++++++++++------------ memory-bank/progress.md | 10 +++++----- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dc7d88..e3ed1ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,8 @@ jobs: run: | docker build -t typst-cv-devcontainer:latest -f .devcontainer/Dockerfile.ubuntu . - - name: Install Bats and File command (for Docker usage tests on host) - run: sudo apt-get update && sudo apt-get install -y bats file + - name: Install Bats, Helpers, and File command (for Docker usage tests on host) + run: sudo apt-get update && sudo apt-get install -y bats bats-support bats-assert file - name: Run internal tests (unit, filter, e2e) inside Devcontainer Image run: | diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index e39a1f3..9b7629b 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,8 +1,8 @@ -# Active Context: Switched Test Dependencies from Submodules to System Packages (2025-04-30) +# Active Context: Resolved CI Test Failures (chmod & symlinks) (2025-04-30) ## Current Focus -Resolving CI failures related to test execution and dependency management. +Verifying CI fixes and completing the task related to switching from submodules to system packages for testing. ## Problem Identified & Resolved @@ -16,17 +16,23 @@ Resolving CI failures related to test execution and dependency management. * Updated Bats test files (`tests/unit/build_sh.bats`, `tests/filter/filters.bats`) to load helpers using `bats_load_library` instead of relative submodule paths. * Updated `justfile` to use the system `bats` executable instead of the submodule path. * Removed the explicit `git submodule update` step from the CI workflow (`.github/workflows/ci.yml`). +6. **New Problem (CI):** Tests run inside the devcontainer image (`just test-internal`) failed with `chmod: changing permissions of '/workspaces/typstCV/build.sh': Operation not permitted`. +7. **Root Cause (CI):** The test script (`tests/unit/build_sh.bats`) attempted to `chmod +x` the `build.sh` script on the host filesystem via the Docker volume mount, which failed due to permissions. Additionally, the symlinks for the local Typst package (`style.typ`, `typst.toml`), created by `postCreateCommand` in the devcontainer, were missing when running the image directly in CI. The host Bats tests (`tests/docker.bats`) were also missing helper packages (`bats-assert`, `bats-support`). +8. **Solution Implemented (CI):** + * Removed the `chmod +x` command from `tests/unit/build_sh.bats`. + * Added `RUN` commands to `.devcontainer/Dockerfile.ubuntu` to create the necessary package symlinks during the image build. + * Updated `.github/workflows/ci.yml` to install `bats-assert` and `bats-support` on the host runner for the `docker.bats` tests. ## Recent Actions (This Session) -- Diagnosed the SSH key failure during `docker build`. -- Discussed options (HTTPS URLs, SSH keys, system packages). -- Decided to switch to system packages for Bats and exclude test/git files from production build context. -- Created `.dockerignore`. -- Removed submodule update step from `.github/workflows/ci.yml`. -- Updated `load` commands in `.bats` files to `bats_load_library`. -- Updated `justfile` to use system `bats`. -- Verified the fix for loading helpers with the user. +- Diagnosed the SSH key failure during `docker build` (previous step). +- Switched test dependencies to system packages and updated related files (previous step). +- Diagnosed the subsequent CI failures (`chmod` error, missing symlinks, missing host Bats helpers). +- Planned the fixes for the CI issues. +- Modified `.devcontainer/Dockerfile.ubuntu` to add symlink creation (and subsequently removed duplicated commands). +- Modified `tests/unit/build_sh.bats` to remove `chmod`. +- Modified `.github/workflows/ci.yml` to install Bats helpers on the host runner. +- Updated this `activeContext.md`. ## Decisions & Notes @@ -37,7 +43,7 @@ Resolving CI failures related to test execution and dependency management. ## Immediate Next Steps - Update `progress.md`. -- **User Action:** Commit the changes (`.dockerignore`, `.github/workflows/ci.yml`, `justfile`, `tests/**/*.bats`). +- **User Action:** Commit the changes (`.devcontainer/Dockerfile.ubuntu`, `tests/unit/build_sh.bats`, `.github/workflows/ci.yml`, and previous changes like `.dockerignore`, `justfile`, other `.bats` files if not already committed). - **User Action:** Trigger the CI workflow and verify all tests pass. -- **User Action (Optional):** Clean up Git submodule configuration (`.gitmodules`, `git rm --cached ...`, `rm -rf ...`). +- **User Action (Optional):** Clean up Git submodule configuration (`.gitmodules`, `git rm --cached tests/bats*`, `rm -rf tests/bats*`). - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 4a8f6fe..f3be25f 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -10,12 +10,12 @@ - Unit tests (`bats`) for `build.sh` logic exist and pass. - Filter tests (`bats`) comparing Pandoc output against snapshots exist and pass. - E2E smoke tests (`sh`) verifying PDF generation for examples exist and pass. - - Docker usage tests (`bats`) verify production container interaction. + - Docker usage tests (`bats`) verify production container interaction (CI host runner now correctly installs helpers). - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.) using system-installed `bats`. - **Docker:** - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. - - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, and system-installed Bats for testing. -- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image, runs Docker usage tests against the production image, builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. + - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats, and now includes built-in symlinks for the local Typst package (`style.typ`, `typst.toml`) to support CI testing. +- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (fixed `chmod` error and missing symlinks), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -38,9 +38,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. +- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error in `test-internal`, missing symlinks in `test-internal`, missing Bats helpers for host tests). - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). -- **Ready for Next Phase:** Project is stable, CI should pass. +- **Ready for Verification:** Project is stable, all known CI issues are addressed. Ready for user to commit changes and trigger CI workflow. ## Known Issues From 3dc3982f7dd28b616a9d8d2c32aed0005cd3ca1e Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 15:02:45 +0000 Subject: [PATCH 49/63] Fix PDF write permission error in Bats tests by directing output to a dedicated subdirectory --- memory-bank/activeContext.md | 9 ++++++++- memory-bank/progress.md | 4 ++-- tests/unit/build_sh.bats | 16 ++++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 9b7629b..006f32c 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -22,6 +22,10 @@ Verifying CI fixes and completing the task related to switching from submodules * Removed the `chmod +x` command from `tests/unit/build_sh.bats`. * Added `RUN` commands to `.devcontainer/Dockerfile.ubuntu` to create the necessary package symlinks during the image build. * Updated `.github/workflows/ci.yml` to install `bats-assert` and `bats-support` on the host runner for the `docker.bats` tests. +9. **New Problem (CI - Post Fixes):** Tests run inside the devcontainer image (`just test-internal`) failed again, this time with `error: failed to write PDF file (Permission denied (os error 13))`. +10. **Root Cause (CI - Post Fixes):** The test `build.sh: succeeds with valid input file` in `tests/unit/build_sh.bats` was attempting to write the output PDF (`dummy.pdf`) directly into the mounted workspace directory (`/workspaces/typstCV`), which the `vscode` user inside the container didn't have permission to do. +11. **Solution Implemented (CI - Post Fixes):** + * Modified the `build.sh: succeeds with valid input file` test in `tests/unit/build_sh.bats` to explicitly write its output PDF to a dedicated subdirectory within the Bats temporary directory (`$BATS_TMPDIR/test_output/`) instead of the current working directory. Added setup/teardown for this directory. ## Recent Actions (This Session) @@ -32,6 +36,8 @@ Verifying CI fixes and completing the task related to switching from submodules - Modified `.devcontainer/Dockerfile.ubuntu` to add symlink creation (and subsequently removed duplicated commands). - Modified `tests/unit/build_sh.bats` to remove `chmod`. - Modified `.github/workflows/ci.yml` to install Bats helpers on the host runner. +- Diagnosed the PDF write permission error (`os error 13`) in CI. +- Modified `tests/unit/build_sh.bats` again to write test output to `$BATS_TMPDIR/test_output/` instead of the CWD. - Updated this `activeContext.md`. ## Decisions & Notes @@ -43,7 +49,8 @@ Verifying CI fixes and completing the task related to switching from submodules ## Immediate Next Steps - Update `progress.md`. -- **User Action:** Commit the changes (`.devcontainer/Dockerfile.ubuntu`, `tests/unit/build_sh.bats`, `.github/workflows/ci.yml`, and previous changes like `.dockerignore`, `justfile`, other `.bats` files if not already committed). +- Update `progress.md`. +- **User Action:** Commit the changes (including the latest update to `tests/unit/build_sh.bats`, `.devcontainer/Dockerfile.ubuntu`, `.github/workflows/ci.yml`, and previous changes like `.dockerignore`, `justfile`, other `.bats` files if not already committed). - **User Action:** Trigger the CI workflow and verify all tests pass. - **User Action (Optional):** Clean up Git submodule configuration (`.gitmodules`, `git rm --cached tests/bats*`, `rm -rf tests/bats*`). - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index f3be25f..6291fc8 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -15,7 +15,7 @@ - **Docker:** - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats, and now includes built-in symlinks for the local Typst package (`style.typ`, `typst.toml`) to support CI testing. -- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (fixed `chmod` error and missing symlinks), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. +- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (fixed `chmod` error, missing symlinks, and PDF write permission error by adjusting test output paths), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -38,7 +38,7 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error in `test-internal`, missing symlinks in `test-internal`, missing Bats helpers for host tests). +- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error in `test-internal`, missing symlinks in `test-internal`, missing Bats helpers for host tests, and PDF write permission error in `test-internal`). - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). - **Ready for Verification:** Project is stable, all known CI issues are addressed. Ready for user to commit changes and trigger CI workflow. diff --git a/tests/unit/build_sh.bats b/tests/unit/build_sh.bats index 8f13fd9..d07f223 100755 --- a/tests/unit/build_sh.bats +++ b/tests/unit/build_sh.bats @@ -5,8 +5,9 @@ bats_load_library 'bats-support' bats_load_library 'bats-assert' setup() { - # Create a dummy input file for tests that need one + # Create dummy input and output directories for tests mkdir -p "$BATS_TMPDIR/fixtures" + mkdir -p "$BATS_TMPDIR/test_output" cat > "$BATS_TMPDIR/fixtures/dummy.md" < Date: Wed, 30 Apr 2025 16:06:00 +0000 Subject: [PATCH 50/63] Setup Pandoc symlinks in Dockerfile and update build.sh to use relative template and Lua filter names --- .devcontainer/Dockerfile.ubuntu | 13 +++++++++++++ build.sh | 2 ++ 2 files changed, 15 insertions(+) diff --git a/.devcontainer/Dockerfile.ubuntu b/.devcontainer/Dockerfile.ubuntu index 0018bbe..88db411 100644 --- a/.devcontainer/Dockerfile.ubuntu +++ b/.devcontainer/Dockerfile.ubuntu @@ -59,6 +59,19 @@ RUN wget -qO typst.tar.xz https://github.com/typst/typst/releases/download/${TYP # Install fontist RUN gem install fontist --no-document --version "~> 1.21.1" +# --- Setup Pandoc Symlinks for Development/Testing --- +# Create standard pandoc directories and symlink workspace files into them +# This makes the devcontainer environment match the production container's setup +# where these files are copied into standard locations. +# Needs to run as root before switching to vscode user. +ENV PANDOC_DATA_DIR=/usr/share/pandoc +RUN mkdir -p $PANDOC_DATA_DIR/filters && \ + mkdir -p $PANDOC_DATA_DIR/templates && \ + ln -sf /workspaces/typstCV/linkify.lua $PANDOC_DATA_DIR/filters/linkify.lua && \ + ln -sf /workspaces/typstCV/typst-cv.lua $PANDOC_DATA_DIR/filters/typst-cv.lua && \ + ln -sf /workspaces/typstCV/typst-cv.typ $PANDOC_DATA_DIR/templates/typst-cv.typ && \ + ln -sf /workspaces/typstCV/typst-letter.typ $PANDOC_DATA_DIR/templates/typst-letter.typ + # Switch back to vscode user provided by base image USER vscode diff --git a/build.sh b/build.sh index 2a59ddd..f741f25 100755 --- a/build.sh +++ b/build.sh @@ -117,6 +117,7 @@ if [ -z "$doc_type" ]; then doc_type="cv" fi fi +# Use relative template filename template_file="typst-${doc_type}.typ" echo "Info: Using document type: $doc_type (template: $template_file)" >&2 @@ -178,6 +179,7 @@ if [ "$input_file" != "-" ]; then fi fi # Add --data-dir to the base command - resource path added per-command +# Use relative Lua filter names # shellcheck disable=SC2089 pandoc_base="pandoc --data-dir $PANDOC_DATA_DIR --wrap=preserve --pdf-engine=typst --lua-filter=linkify.lua --lua-filter=typst-cv.lua" From f789c3929c4e99c25feffac80e9c2c3e19f550ff Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 16:07:45 +0000 Subject: [PATCH 51/63] Refactor E2E tests: replace shell script with Bats tests for PDF generation and update Justfile for new test structure --- justfile | 6 +- tests/test_e2e.bats | 133 ++++++++++++++++++++++++++++++++++++++ tests/test_e2e.sh | 151 -------------------------------------------- 3 files changed, 136 insertions(+), 154 deletions(-) create mode 100755 tests/test_e2e.bats delete mode 100755 tests/test_e2e.sh diff --git a/justfile b/justfile index c430758..564898d 100644 --- a/justfile +++ b/justfile @@ -3,8 +3,8 @@ # Variables bats_executable := "bats" # Use system-installed bats unit_tests_file := "tests/unit/build_sh.bats" -filter_tests_file := "tests/filter/filters.bats" # Updated path -e2e_tests_script := "tests/test_e2e.sh" +filter_tests_file := "tests/filter/filters.bats" +e2e_tests_file := "tests/test_e2e.bats" # Changed from script to file # Default task default: @@ -24,7 +24,7 @@ test-filter: # Run E2E smoke tests (PDF generation) test-e2e: - sh {{e2e_tests_script}} + {{bats_executable}} {{e2e_tests_file}} # Use bats now # Run Docker usage tests (requires image built, e.g., via `just build-docker`) test-docker: diff --git a/tests/test_e2e.bats b/tests/test_e2e.bats new file mode 100755 index 0000000..7e95b41 --- /dev/null +++ b/tests/test_e2e.bats @@ -0,0 +1,133 @@ +#!/usr/bin/env bats + +# Load Bats helpers +bats_load_library 'bats-support' +bats_load_library 'bats-assert' +# bats_load_library 'bats-file' # Can add later if needed + +# --- Variables --- +PROJECT_ROOT="" +SCRIPT="" +FIXTURES_DIR="" +TEST_OUTPUT_DIR="" + +# --- Setup & Teardown --- +setup() { + # Determine project root relative to this test file's directory + PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/.." && pwd)" + SCRIPT="$PROJECT_ROOT/build.sh" + FIXTURES_DIR="$PROJECT_ROOT/tests/fixtures" + TEST_OUTPUT_DIR="$BATS_TMPDIR/test_output" + + # Create temporary output directory + mkdir -p "$TEST_OUTPUT_DIR" + + # Ensure build script is executable (should be via git, but double-check) + # No - this caused issues before. Assume it's executable. + # chmod +x "$SCRIPT" +} + +teardown() { + # Clean up temporary output directory + rm -rf "$TEST_OUTPUT_DIR" +} + +# --- Test Cases --- + +@test "e2e: generates PDF for example-cv.md" { + local input_fixture="$FIXTURES_DIR/example-cv.md" + local output_filename="example-cv.pdf" + local output_file_abs="$TEST_OUTPUT_DIR/$output_filename" + + # Run build.sh from /tmp to avoid pandoc permission issues on mounted volume + # Pass absolute paths for input/output-dir + run bash -c "cd \"$BATS_TMPDIR\" && \"$SCRIPT\" --output-dir \"$TEST_OUTPUT_DIR\" --output \"$output_filename\" \"$input_fixture\"" + assert_success "[build.sh] PDF generation failed" + + run test -f "$output_file_abs" + assert_success "Output PDF file not found: $output_file_abs" + run test -s "$output_file_abs" + assert_success "Output PDF file is empty: $output_file_abs" +} + +@test "e2e: example-cv.md PDF contains hidden headings" { + local input_fixture="$FIXTURES_DIR/example-cv.md" + local output_filename="example-cv-headings.pdf" # Use different name to avoid conflict if run in parallel + local output_file_abs="$TEST_OUTPUT_DIR/$output_filename" + + # Generate the PDF first (run from /tmp) + run bash -c "cd \"$BATS_TMPDIR\" && \"$SCRIPT\" --output-dir \"$TEST_OUTPUT_DIR\" --output \"$output_filename\" \"$input_fixture\"" + assert_success "[build.sh] PDF generation for heading check failed" + + run test -f "$output_file_abs" + assert_success "Output PDF file for heading check not found: $output_file_abs" + + # Check for pdftotext + run command -v pdftotext + if [ "$status" -ne 0 ]; then + skip "pdftotext not found, skipping content check" + fi + + # Extract text and check content + run pdftotext "$output_file_abs" - + assert_success "pdftotext failed" + assert_output --partial "Experience" "Hidden heading 'Experience' not found" + assert_output --partial "Education" "Hidden heading 'Education' not found" +} + +@test "e2e: generates PDF for example-letter.md" { + local input_fixture="$FIXTURES_DIR/example-letter.md" + local output_filename="example-letter.pdf" + local output_file_abs="$TEST_OUTPUT_DIR/$output_filename" + + # Run build.sh from /tmp, pass --type letter + run bash -c "cd \"$BATS_TMPDIR\" && \"$SCRIPT\" --output-dir \"$TEST_OUTPUT_DIR\" --output \"$output_filename\" \"$input_fixture\" --type letter" + assert_success "[build.sh] PDF generation failed for letter" + + run test -f "$output_file_abs" + assert_success "Output letter PDF file not found: $output_file_abs" + run test -s "$output_file_abs" + assert_success "Output letter PDF file is empty: $output_file_abs" +} + +@test "e2e: generates PDF for example-cv-no-optional.md" { + local input_fixture="$FIXTURES_DIR/example-cv-no-optional.md" + local output_filename="example-cv-no-optional.pdf" + local output_file_abs="$TEST_OUTPUT_DIR/$output_filename" + + # Run build.sh from /tmp + run bash -c "cd \"$BATS_TMPDIR\" && \"$SCRIPT\" --output-dir \"$TEST_OUTPUT_DIR\" --output \"$output_filename\" \"$input_fixture\"" + assert_success "[build.sh] PDF generation failed for no-optional CV" + + run test -f "$output_file_abs" + assert_success "Output no-optional CV PDF file not found: $output_file_abs" + run test -s "$output_file_abs" + assert_success "Output no-optional CV PDF file is empty: $output_file_abs" +} + +@test "e2e: example-cv-no-optional.md PDF lacks footer links" { + local input_fixture="$FIXTURES_DIR/example-cv-no-optional.md" + local output_filename="example-cv-no-optional-links.pdf" # Use different name + local output_file_abs="$TEST_OUTPUT_DIR/$output_filename" + + # Generate the PDF first (run from /tmp) + run bash -c "cd \"$BATS_TMPDIR\" && \"$SCRIPT\" --output-dir \"$TEST_OUTPUT_DIR\" --output \"$output_filename\" \"$input_fixture\"" + assert_success "[build.sh] PDF generation for no-optional link check failed" + + run test -f "$output_file_abs" + assert_success "Output PDF file for no-optional link check not found: $output_file_abs" + + # Check for pdftotext + run command -v pdftotext + if [ "$status" -ne 0 ]; then + skip "pdftotext not found, skipping content check" + fi + + # Extract text and check content + run pdftotext "$output_file_abs" - + assert_success "pdftotext failed" + # Check that specific URL parts or usernames are NOT present + refute_output --partial "github.com/" "Found 'github.com/' in no-optional PDF" + refute_output --partial "gitlab.com/" "Found 'gitlab.com/' in no-optional PDF" + refute_output --partial "linkedin.com/in/" "Found 'linkedin.com/in/' in no-optional PDF" +} diff --git a/tests/test_e2e.sh b/tests/test_e2e.sh deleted file mode 100755 index f226810..0000000 --- a/tests/test_e2e.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/sh -set -e - -# E2E smoke test script for build.sh (generating PDF) - -# --- Configuration --- -FIXTURES_DIR="./tests/fixtures" -BUILD_SCRIPT="./build.sh" -TEST_OUTPUT_DIR=$(mktemp -d) - -echo "--- Running E2E Smoke Tests (PDF Output) ---" - -# shellcheck disable=SC2317 # Function appears unreachable but is called by trap -cleanup() { - echo "Cleaning up temporary directory: $TEST_OUTPUT_DIR" - rm -rf "$TEST_OUTPUT_DIR" -} -trap cleanup EXIT # Ensure cleanup happens on script exit - -# --- Test example-cv.md PDF Generation --- -echo "Testing example-cv.md -> PDF..." -CV_FIXTURE="${FIXTURES_DIR}/example-cv.md" -CV_EXPECTED_FILENAME="example-cv.pdf" -CV_EXPECTED_PDF="${TEST_OUTPUT_DIR}/${CV_EXPECTED_FILENAME}" - -# Run build.sh to generate PDF output, specifying output dir and filename -"$BUILD_SCRIPT" --output-dir "$TEST_OUTPUT_DIR" --output "$CV_EXPECTED_FILENAME" "$CV_FIXTURE" -exit_code=$? - -if [ $exit_code -ne 0 ]; then - echo "Error: build.sh failed for example-cv.md PDF generation" >&2 - exit 1 -fi - -# Check if PDF exists and has size > 0 -if [ ! -f "$CV_EXPECTED_PDF" ]; then - echo "Error: Expected PDF file not found: $CV_EXPECTED_PDF" >&2 - exit 1 -fi -if [ ! -s "$CV_EXPECTED_PDF" ]; then - echo "Error: Generated PDF file is empty: $CV_EXPECTED_PDF" >&2 - exit 1 -fi -echo "example-cv.md PDF generated successfully." - -# --- Check example-cv.md PDF Content (Hidden Headings) --- -echo "Checking example-cv.md PDF text content..." -CV_TXT_OUTPUT=$(mktemp) -if command -v pdftotext >/dev/null 2>&1; then - pdftotext "$CV_EXPECTED_PDF" "$CV_TXT_OUTPUT" - # Check that hidden headings ARE present in the text layer - if ! grep -q "Experience" "$CV_TXT_OUTPUT"; then - echo "Error: Hidden heading 'Experience' not found in PDF text output." >&2 - cat "$CV_TXT_OUTPUT" >&2 - rm "$CV_TXT_OUTPUT" - exit 1 - fi - if ! grep -q "Education" "$CV_TXT_OUTPUT"; then - echo "Error: Hidden heading 'Education' not found in PDF text output." >&2 - cat "$CV_TXT_OUTPUT" >&2 - rm "$CV_TXT_OUTPUT" - exit 1 - fi - rm "$CV_TXT_OUTPUT" - echo "Hidden heading checks passed." -else - echo "Warning: pdftotext not found. Skipping hidden heading content check." >&2 -fi - - -# --- Test example-letter.md PDF Generation --- -echo "Testing example-letter.md -> PDF..." -LETTER_FIXTURE="${FIXTURES_DIR}/example-letter.md" -LETTER_EXPECTED_FILENAME="example-letter.pdf" -LETTER_EXPECTED_PDF="${TEST_OUTPUT_DIR}/${LETTER_EXPECTED_FILENAME}" - -# Run build.sh to generate PDF output, specifying output dir and filename -"$BUILD_SCRIPT" --output-dir "$TEST_OUTPUT_DIR" --output "$LETTER_EXPECTED_FILENAME" "$LETTER_FIXTURE" --type letter -exit_code=$? - -if [ $exit_code -ne 0 ]; then - echo "Error: build.sh failed for example-letter.md PDF generation" >&2 - exit 1 -fi - -# Check if PDF exists and has size > 0 -if [ ! -f "$LETTER_EXPECTED_PDF" ]; then - echo "Error: Expected PDF file not found: $LETTER_EXPECTED_PDF" >&2 - exit 1 -fi -if [ ! -s "$LETTER_EXPECTED_PDF" ]; then - echo "Error: Generated PDF file is empty: $LETTER_EXPECTED_PDF" >&2 - exit 1 -fi -echo "example-letter.md PDF generated successfully." - - -# --- Test example-cv-no-optional.md PDF Generation (Check Footer) --- -echo "Testing example-cv-no-optional.md -> PDF (No Optional Footer Links)..." -NOOPT_FIXTURE="${FIXTURES_DIR}/example-cv-no-optional.md" -NOOPT_EXPECTED_FILENAME="example-cv-no-optional.pdf" -NOOPT_EXPECTED_PDF="${TEST_OUTPUT_DIR}/${NOOPT_EXPECTED_FILENAME}" -NOOPT_TXT_OUTPUT=$(mktemp) - -# Run build.sh to generate PDF output -"$BUILD_SCRIPT" --output-dir "$TEST_OUTPUT_DIR" --output "$NOOPT_EXPECTED_FILENAME" "$NOOPT_FIXTURE" -exit_code=$? - -if [ $exit_code -ne 0 ]; then - echo "Error: build.sh failed for example-cv-no-optional.md PDF generation" >&2 - exit 1 -fi - -# Check if PDF exists and has size > 0 -if [ ! -f "$NOOPT_EXPECTED_PDF" ]; then - echo "Error: Expected PDF file not found: $NOOPT_EXPECTED_PDF" >&2 - exit 1 -fi -if [ ! -s "$NOOPT_EXPECTED_PDF" ]; then - echo "Error: Generated PDF file is empty: $NOOPT_EXPECTED_PDF" >&2 - exit 1 -fi - -# Convert PDF to text and check for absence of optional links -# Requires poppler-utils (pdftotext) to be installed -if command -v pdftotext >/dev/null 2>&1; then - pdftotext "$NOOPT_EXPECTED_PDF" "$NOOPT_TXT_OUTPUT" - # Check that specific URL parts or usernames are NOT present - if grep -q -e "github.com/" -e "gitlab.com/" -e "linkedin.com/in/" "$NOOPT_TXT_OUTPUT"; then - echo "Error: Optional footer links (GitHub/GitLab/LinkedIn) were found in PDF text output when they should be absent." >&2 - cat "$NOOPT_TXT_OUTPUT" >&2 - rm "$NOOPT_TXT_OUTPUT" - exit 1 - fi - # Check for website (might be harder if URL is generic like example.com) - # Let's assume the website field itself shouldn't appear if omitted - if grep -q -i "example.com" "$NOOPT_TXT_OUTPUT"; then - # This check might be too broad, but let's try it. The fixture uses example.com in links: section too. - # A better check might be needed if this causes false positives. - # echo "Warning: Found 'example.com', potentially the website link?" >&2 - : # Placeholder, maybe refine later - fi - rm "$NOOPT_TXT_OUTPUT" - echo "example-cv-no-optional.md PDF checks passed." -else - echo "Warning: pdftotext not found. Skipping footer content check for example-cv-no-optional.md." >&2 -fi - - -echo "--- E2E Smoke Tests Passed ---" -exit 0 From f142119d630b59677dfcb39429f284749179cf52 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 16:08:03 +0000 Subject: [PATCH 52/63] Update activeContext and progress documentation to reflect CI test resolutions and E2E test standardization --- memory-bank/activeContext.md | 36 +++++++++++++++++++++++++++--------- memory-bank/progress.md | 22 +++++++++++----------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 006f32c..3dd7473 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,8 +1,8 @@ -# Active Context: Resolved CI Test Failures (chmod & symlinks) (2025-04-30) +# Active Context: Resolved CI Test Failures & Standardized E2E Tests (2025-04-30) ## Current Focus -Verifying CI fixes and completing the task related to switching from submodules to system packages for testing. +Completing the task related to switching from submodules to system packages for testing and resolving subsequent CI issues. ## Problem Identified & Resolved @@ -26,6 +26,15 @@ Verifying CI fixes and completing the task related to switching from submodules 10. **Root Cause (CI - Post Fixes):** The test `build.sh: succeeds with valid input file` in `tests/unit/build_sh.bats` was attempting to write the output PDF (`dummy.pdf`) directly into the mounted workspace directory (`/workspaces/typstCV`), which the `vscode` user inside the container didn't have permission to do. 11. **Solution Implemented (CI - Post Fixes):** * Modified the `build.sh: succeeds with valid input file` test in `tests/unit/build_sh.bats` to explicitly write its output PDF to a dedicated subdirectory within the Bats temporary directory (`$BATS_TMPDIR/test_output/`) instead of the current working directory. Added setup/teardown for this directory. +12. **New Problem (CI - E2E):** After fixing the unit tests, the E2E tests (`tests/test_e2e.sh`) failed with `pandoc: .: openTempFile: permission denied` and `Could not find data file ... typst-letter.typ`. +13. **Root Cause (CI - E2E):** The E2E test ran `build.sh` from the mounted workspace, but `build.sh` couldn't find templates/filters relative to the CWD, and Pandoc couldn't create temp files in the CWD. Also, the E2E test used a shell script instead of Bats. +14. **Final Solution Implemented:** + * Converted `tests/test_e2e.sh` to `tests/test_e2e.bats` for consistency. + * Modified `tests/test_e2e.bats` to run `build.sh` from within `$BATS_TMPDIR` to resolve Pandoc's temp file permission issue. + * Modified `.devcontainer/Dockerfile.ubuntu` to create symlinks for Pandoc templates (`*.typ`) and filters (`*.lua`) from the workspace into the standard system locations (`/usr/share/pandoc/...`), aligning the devcontainer environment with the production container. + * Reverted `build.sh` to use relative template/filter names, relying on Pandoc's `--data-dir` mechanism (which now works in both environments). + * Updated `justfile` to run the new `tests/test_e2e.bats`. + * User deleted the old `tests/test_e2e.sh`. ## Recent Actions (This Session) @@ -33,24 +42,33 @@ Verifying CI fixes and completing the task related to switching from submodules - Switched test dependencies to system packages and updated related files (previous step). - Diagnosed the subsequent CI failures (`chmod` error, missing symlinks, missing host Bats helpers). - Planned the fixes for the CI issues. -- Modified `.devcontainer/Dockerfile.ubuntu` to add symlink creation (and subsequently removed duplicated commands). +- Modified `.devcontainer/Dockerfile.ubuntu` to add Typst package symlink creation (and subsequently removed duplicated commands). - Modified `tests/unit/build_sh.bats` to remove `chmod`. - Modified `.github/workflows/ci.yml` to install Bats helpers on the host runner. -- Diagnosed the PDF write permission error (`os error 13`) in CI. +- Diagnosed the PDF write permission error (`os error 13`) in CI unit tests. - Modified `tests/unit/build_sh.bats` again to write test output to `$BATS_TMPDIR/test_output/` instead of the CWD. +- Diagnosed E2E test failures (Pandoc temp file permission, missing templates/filters). +- Converted `tests/test_e2e.sh` to `tests/test_e2e.bats`, incorporating `cd $BATS_TMPDIR` fix and correcting assertion syntax. +- Updated `justfile` to use `tests/test_e2e.bats`. +- Modified `build.sh` to use absolute paths for templates/filters (later reverted). +- Diagnosed Docker test failures caused by absolute paths in `build.sh`. +- Reverted `build.sh` to use relative paths for templates/filters. +- Modified `.devcontainer/Dockerfile.ubuntu` to add symlinks for Pandoc templates/filters to system locations. +- User confirmed tests pass locally after rebuilding devcontainer. +- User deleted `tests/test_e2e.sh`. - Updated this `activeContext.md`. ## Decisions & Notes -- Using system packages for test dependencies simplifies the CI build process and avoids SSH key issues during production image builds. -- `.dockerignore` is crucial for keeping production build contexts clean and secure. -- `bats_load_library` is the correct way to load system-installed Bats helpers. +- Using system packages for test dependencies simplifies the CI build process. +- `.dockerignore` is crucial for keeping production build contexts clean. +- Aligning devcontainer resource locations (via symlinks) with production container locations (via copy) simplifies build scripts and testing. +- Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. ## Immediate Next Steps - Update `progress.md`. -- Update `progress.md`. -- **User Action:** Commit the changes (including the latest update to `tests/unit/build_sh.bats`, `.devcontainer/Dockerfile.ubuntu`, `.github/workflows/ci.yml`, and previous changes like `.dockerignore`, `justfile`, other `.bats` files if not already committed). +- **User Action:** Commit the changes (including `tests/test_e2e.bats`, `justfile`, `.devcontainer/Dockerfile.ubuntu`, `build.sh`, `tests/unit/build_sh.bats`, `.github/workflows/ci.yml`, `.dockerignore`, and Memory Bank files). Ensure `tests/test_e2e.sh` is deleted. - **User Action:** Trigger the CI workflow and verify all tests pass. - **User Action (Optional):** Clean up Git submodule configuration (`.gitmodules`, `git rm --cached tests/bats*`, `rm -rf tests/bats*`). - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 6291fc8..e7aa1ac 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -7,15 +7,15 @@ - **Features:** Automatic linkification (`linkify.lua`), CV side content (`{photo}`, `{location}`, `{date}` via `typst-cv.lua`), hidden headings (`.hidden`), and full-width ordered lists are functional. - **Examples:** Generic, depersonalized example files (`example-cv.md`, `example-letter.md`) and placeholder image exist in `tests/fixtures/`. - **Testing:** - - Unit tests (`bats`) for `build.sh` logic exist and pass. - - Filter tests (`bats`) comparing Pandoc output against snapshots exist and pass. - - E2E smoke tests (`sh`) verifying PDF generation for examples exist and pass. - - Docker usage tests (`bats`) verify production container interaction (CI host runner now correctly installs helpers). - - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, etc.) using system-installed `bats`. + - Unit tests (`bats`) for `build.sh` logic exist and pass (`tests/unit/build_sh.bats`). + - Filter tests (`bats`) comparing Pandoc output against snapshots exist and pass (`tests/filter/filters.bats`). + - E2E smoke tests converted to Bats (`tests/test_e2e.bats`), verifying PDF generation and basic content. Uses `cd $BATS_TMPDIR` workaround for Pandoc temp file permissions. + - Docker usage tests (`bats`) verify production container interaction (`tests/docker.bats`). CI host runner now correctly installs helpers. + - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, `just test-filter`, `just test-e2e`) using system-installed `bats`. - **Docker:** - - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. - - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats, and now includes built-in symlinks for the local Typst package (`style.typ`, `typst.toml`) to support CI testing. -- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (fixed `chmod` error, missing symlinks, and PDF write permission error by adjusting test output paths), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. + - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. Files (`*.lua`, `*.typ`) copied to standard Pandoc/Typst locations. + - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats. Includes built-in symlinks for the local Typst package (`style.typ`, `typst.toml`) *and* for Pandoc templates/filters (`*.typ`, `*.lua`) into standard system locations, aligning it with the production environment. +- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (all known permission/path issues resolved), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -38,9 +38,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error in `test-internal`, missing symlinks in `test-internal`, missing Bats helpers for host tests, and PDF write permission error in `test-internal`). -- **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). -- **Ready for Verification:** Project is stable, all known CI issues are addressed. Ready for user to commit changes and trigger CI workflow. +- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error, missing Typst package symlinks, PDF write permission error, missing host Bats helpers, Pandoc temp file permissions, Pandoc template/filter path issues) by modifying tests, Dockerfiles, and build script as needed. Standardized E2E tests to Bats. +- **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. +- **Ready for Verification:** Project is stable, all known CI issues are addressed, tests pass locally. Ready for user to commit changes and trigger CI workflow for final confirmation. ## Known Issues From 56f9ad30dbd1d1dce6f6cfb35c8a9f71a309b2eb Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 16:40:29 +0000 Subject: [PATCH 53/63] Fix CI Docker tests by parameterizing image tag and updating workflow --- .github/workflows/ci.yml | 3 ++- memory-bank/activeContext.md | 12 +++++++++--- memory-bank/progress.md | 6 +++--- tests/docker.bats | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3ed1ad..93095d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,8 @@ jobs: # These tests run on the host, interacting with the built testing image # Assumes 'bats' is installed on the runner (installed in previous step) # Assumes docker is available on the runner - bats ${{ env.TEST_DIR }}/docker.bats + # Pass the correct image tag to the test script via env var + DOCKER_IMAGE_TAG=${{ env.IMAGE_TAG_TESTING }} bats ${{ env.TEST_DIR }}/docker.bats echo "Docker usage tests finished." - name: Build Example PDFs for Release Artifacts (using production image, mounting only fixtures) diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 3dd7473..5973e87 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -35,6 +35,11 @@ Completing the task related to switching from submodules to system packages for * Reverted `build.sh` to use relative template/filter names, relying on Pandoc's `--data-dir` mechanism (which now works in both environments). * Updated `justfile` to run the new `tests/test_e2e.bats`. * User deleted the old `tests/test_e2e.sh`. +15. **New Problem (CI - Docker Tests):** The `test-docker` step failed with `!!! Docker image 'typst-cv:latest' not found`. +16. **Root Cause (CI - Docker Tests):** The test script (`tests/docker.bats`) hardcoded the image tag `typst-cv:latest`, but the CI workflow built and provided the image tagged as `kadykov/typst-cv:testing`. +17. **Solution Implemented (CI - Docker Tests):** + * Modified `tests/docker.bats` to read the image tag from an environment variable (`DOCKER_IMAGE_TAG`), defaulting to `typst-cv:latest` if unset. + * Modified `.github/workflows/ci.yml` to pass the correct tag (`${{ env.IMAGE_TAG_TESTING }}`) to the `bats` command via the `DOCKER_IMAGE_TAG` environment variable. ## Recent Actions (This Session) @@ -56,6 +61,7 @@ Completing the task related to switching from submodules to system packages for - Modified `.devcontainer/Dockerfile.ubuntu` to add symlinks for Pandoc templates/filters to system locations. - User confirmed tests pass locally after rebuilding devcontainer. - User deleted `tests/test_e2e.sh`. +- Diagnosed and fixed the `test-docker` CI failure by parameterizing the image tag. - Updated this `activeContext.md`. ## Decisions & Notes @@ -64,11 +70,11 @@ Completing the task related to switching from submodules to system packages for - `.dockerignore` is crucial for keeping production build contexts clean. - Aligning devcontainer resource locations (via symlinks) with production container locations (via copy) simplifies build scripts and testing. - Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. +- Using an environment variable (`DOCKER_IMAGE_TAG`) allows the Docker usage tests (`tests/docker.bats`) to work correctly in both local (defaulting to `typst-cv:latest`) and CI environments (using the specific testing tag). ## Immediate Next Steps - Update `progress.md`. -- **User Action:** Commit the changes (including `tests/test_e2e.bats`, `justfile`, `.devcontainer/Dockerfile.ubuntu`, `build.sh`, `tests/unit/build_sh.bats`, `.github/workflows/ci.yml`, `.dockerignore`, and Memory Bank files). Ensure `tests/test_e2e.sh` is deleted. -- **User Action:** Trigger the CI workflow and verify all tests pass. -- **User Action (Optional):** Clean up Git submodule configuration (`.gitmodules`, `git rm --cached tests/bats*`, `rm -rf tests/bats*`). +- **User Action:** Commit the changes (including `tests/docker.bats`, `.github/workflows/ci.yml`, and Memory Bank files). +- **User Action:** Trigger the CI workflow and verify all tests pass, including the `test-docker` step. - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index e7aa1ac..e732ea4 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -10,7 +10,7 @@ - Unit tests (`bats`) for `build.sh` logic exist and pass (`tests/unit/build_sh.bats`). - Filter tests (`bats`) comparing Pandoc output against snapshots exist and pass (`tests/filter/filters.bats`). - E2E smoke tests converted to Bats (`tests/test_e2e.bats`), verifying PDF generation and basic content. Uses `cd $BATS_TMPDIR` workaround for Pandoc temp file permissions. - - Docker usage tests (`bats`) verify production container interaction (`tests/docker.bats`). CI host runner now correctly installs helpers. + - Docker usage tests (`bats`) verify production container interaction (`tests/docker.bats`). CI host runner correctly installs helpers. **CI execution fixed** by passing the correct image tag via `DOCKER_IMAGE_TAG` environment variable. - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, `just test-filter`, `just test-e2e`) using system-installed `bats`. - **Docker:** - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. Files (`*.lua`, `*.typ`) copied to standard Pandoc/Typst locations. @@ -38,9 +38,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error, missing Typst package symlinks, PDF write permission error, missing host Bats helpers, Pandoc temp file permissions, Pandoc template/filter path issues) by modifying tests, Dockerfiles, and build script as needed. Standardized E2E tests to Bats. +- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error, missing Typst package symlinks, PDF write permission error, missing host Bats helpers, Pandoc temp file permissions, Pandoc template/filter path issues, **Docker usage test image tag mismatch**) by modifying tests, Dockerfiles, build script, and CI workflow as needed. Standardized E2E tests to Bats. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable, all known CI issues are addressed, tests pass locally. Ready for user to commit changes and trigger CI workflow for final confirmation. +- **Ready for Verification:** Project is stable, all known CI issues are addressed (including `test-docker`), tests pass locally. Ready for user to commit changes and trigger CI workflow for final confirmation. ## Known Issues diff --git a/tests/docker.bats b/tests/docker.bats index 36b7dd5..ce1f142 100755 --- a/tests/docker.bats +++ b/tests/docker.bats @@ -5,8 +5,8 @@ bats_load_library 'bats-support' bats_load_library 'bats-assert' # --- Variables --- -# Assumes the image is built with the 'latest' tag locally -DOCKER_IMAGE="typst-cv:latest" +# Use DOCKER_IMAGE_TAG env var if set (e.g., in CI), otherwise default to 'typst-cv:latest' for local runs +DOCKER_IMAGE="${DOCKER_IMAGE_TAG:-typst-cv:latest}" FIXTURES_DIR="tests/fixtures" # Relative path inside the devcontainer OUTPUT_DIR_TMP="" From dc0d96831ecafdeb103cb0baf8c72e42375a4bc7 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 18:22:51 +0000 Subject: [PATCH 54/63] Fix CI workflow for building example PDFs by correcting entrypoint argument handling --- .github/workflows/ci.yml | 5 +++-- memory-bank/activeContext.md | 10 ++++++++-- memory-bank/progress.md | 6 +++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93095d6..f2f243a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,10 +101,11 @@ jobs: # Create host output directory first mkdir -p ./examples # Mount fixtures to /test-fixtures, mount host output dir to /output + # Pass arguments directly to the ENTRYPOINT (build.sh) docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" -v "${PWD}/examples:/output" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - build.sh --output-dir /output /test-fixtures/example-cv.md + --output-dir /output /test-fixtures/example-cv.md docker run --rm -v "${PWD}/${{ env.TEST_DIR }}/fixtures:/test-fixtures:ro" -v "${PWD}/examples:/output" --workdir /data ${{ env.IMAGE_TAG_TESTING }} \ - build.sh --output-dir /output /test-fixtures/example-letter.md --type letter + --output-dir /output /test-fixtures/example-letter.md --type letter echo "Example PDFs built." - name: Upload Example PDFs diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 5973e87..bf25c74 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -40,6 +40,10 @@ Completing the task related to switching from submodules to system packages for 17. **Solution Implemented (CI - Docker Tests):** * Modified `tests/docker.bats` to read the image tag from an environment variable (`DOCKER_IMAGE_TAG`), defaulting to `typst-cv:latest` if unset. * Modified `.github/workflows/ci.yml` to pass the correct tag (`${{ env.IMAGE_TAG_TESTING }}`) to the `bats` command via the `DOCKER_IMAGE_TAG` environment variable. +18. **New Problem (CI - Example Build):** The "Build Example PDFs for Release Artifacts" step failed with `Error: Only one input file allowed.` +19. **Root Cause (CI - Example Build):** The `docker run` commands in the CI workflow incorrectly included `build.sh` as part of the command being passed to the container. Since the container has `ENTRYPOINT ["build.sh"]`, the script name was passed as the first argument to the entrypoint script itself, leading to the argument parsing error. +20. **Solution Implemented (CI - Example Build):** + * Modified `.github/workflows/ci.yml` to remove the redundant `build.sh` from the `docker run` commands in the "Build Example PDFs" step, ensuring only the intended arguments are passed to the entrypoint. ## Recent Actions (This Session) @@ -62,6 +66,7 @@ Completing the task related to switching from submodules to system packages for - User confirmed tests pass locally after rebuilding devcontainer. - User deleted `tests/test_e2e.sh`. - Diagnosed and fixed the `test-docker` CI failure by parameterizing the image tag. +- Diagnosed and fixed the "Build Example PDFs" CI failure by correcting the arguments passed to the Docker entrypoint. - Updated this `activeContext.md`. ## Decisions & Notes @@ -71,10 +76,11 @@ Completing the task related to switching from submodules to system packages for - Aligning devcontainer resource locations (via symlinks) with production container locations (via copy) simplifies build scripts and testing. - Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. - Using an environment variable (`DOCKER_IMAGE_TAG`) allows the Docker usage tests (`tests/docker.bats`) to work correctly in both local (defaulting to `typst-cv:latest`) and CI environments (using the specific testing tag). +- When using `docker run` with an `ENTRYPOINT`, the command specified after the image name is passed as arguments *to* the entrypoint script. ## Immediate Next Steps - Update `progress.md`. -- **User Action:** Commit the changes (including `tests/docker.bats`, `.github/workflows/ci.yml`, and Memory Bank files). -- **User Action:** Trigger the CI workflow and verify all tests pass, including the `test-docker` step. +- **User Action:** Commit the changes (including `.github/workflows/ci.yml` and Memory Bank files). +- **User Action:** Trigger the CI workflow and verify all tests pass, including the "Build Example PDFs" step. - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index e732ea4..04be157 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -15,7 +15,7 @@ - **Docker:** - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. Files (`*.lua`, `*.typ`) copied to standard Pandoc/Typst locations. - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats. Includes built-in symlinks for the local Typst package (`style.typ`, `typst.toml`) *and* for Pandoc templates/filters (`*.typ`, `*.lua`) into standard system locations, aligning it with the production environment. -- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (all known permission/path issues resolved), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container, pushes images, and handles releases. Submodule handling removed. +- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (all known permission/path issues resolved), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container (**fixed entrypoint argument issue**), pushes images, and handles releases. Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -38,9 +38,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error, missing Typst package symlinks, PDF write permission error, missing host Bats helpers, Pandoc temp file permissions, Pandoc template/filter path issues, **Docker usage test image tag mismatch**) by modifying tests, Dockerfiles, build script, and CI workflow as needed. Standardized E2E tests to Bats. +- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error, missing Typst package symlinks, PDF write permission error, missing host Bats helpers, Pandoc temp file permissions, Pandoc template/filter path issues, **Docker usage test image tag mismatch**, **Build Example PDFs entrypoint argument mismatch**) by modifying tests, Dockerfiles, build script, and CI workflow as needed. Standardized E2E tests to Bats. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable, all known CI issues are addressed (including `test-docker`), tests pass locally. Ready for user to commit changes and trigger CI workflow for final confirmation. +- **Ready for Verification:** Project is stable, all known CI issues are addressed (including `test-docker` and example builds), tests pass locally. Ready for user to commit changes and trigger CI workflow for final confirmation. ## Known Issues From b731a9174c519c0379c970b664febecc4e80bd97 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 22:35:28 +0000 Subject: [PATCH 55/63] Enhance CI workflow: add devcontainer caching to GHCR, implement conditional production image push, and update Docker login steps --- .github/workflows/ci.yml | 47 ++++++++++++++++++++++++++++++------ memory-bank/activeContext.md | 41 +++++++++++++------------------ memory-bank/progress.md | 19 +++++++++++---- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2f243a..fe0aad9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ env: OUTPUT_ARTIFACTS_NAME: pdf EXAMPLE_ARTIFACTS_NAME: examples TEST_DIR: tests # Directory containing test scripts and fixtures + DEV_IMAGE_NAME: ghcr.io/${{ github.repository }}/devcontainer jobs: pre_commit: @@ -52,7 +53,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry # For devcontainer cache + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub # For production image testing cache pull uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} @@ -71,18 +79,37 @@ jobs: # Submodules are no longer used for testing dependencies # submodules: recursive # Keep commented out as reference if needed - - name: Build Devcontainer Image for Testing - run: | - docker build -t typst-cv-devcontainer:latest -f .devcontainer/Dockerfile.ubuntu . + - name: Devcontainer Docker meta # For devcontainer cache image + id: devmeta # Ensure this ID is unique + uses: docker/metadata-action@v5 + with: + images: ${{ env.DEV_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and Cache Devcontainer Image + id: build_devcontainer + uses: docker/build-push-action@v6 + with: + context: . + file: .devcontainer/Dockerfile.ubuntu + tags: ${{ steps.devmeta.outputs.tags }} + push: true # Always push cache to GHCR + cache-from: type=registry,ref=${{ env.DEV_IMAGE_NAME }}:latest + cache-to: type=registry,ref=${{ env.DEV_IMAGE_NAME }}:latest,mode=max # Cache to latest tag - name: Install Bats, Helpers, and File command (for Docker usage tests on host) run: sudo apt-get update && sudo apt-get install -y bats bats-support bats-assert file + - name: Pull Cached Devcontainer Image + run: docker pull ${{ env.DEV_IMAGE_NAME }}:latest + - name: Run internal tests (unit, filter, e2e) inside Devcontainer Image run: | echo "Running internal tests..." - # Run using the devcontainer image, mounting the full workspace - docker run --rm -v "${PWD}:/workspaces/typstCV" --workdir /workspaces/typstCV typst-cv-devcontainer:latest just test-internal + # Run using the pulled devcontainer image from GHCR, mounting the full workspace + docker run --rm -v "${PWD}:/workspaces/typstCV" --workdir /workspaces/typstCV ${{ env.DEV_IMAGE_NAME }}:latest just test-internal echo "Internal tests finished." - name: Run Docker usage tests (on runner host against production image) @@ -133,15 +160,19 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} + type=ref,event=pr - - name: Build and push + - name: Build and push # Production image to Docker Hub + if: success() && (github.event_name == 'push') # Only push on push events uses: docker/build-push-action@v6 with: - push: true + push: true # This step only runs on push events anyway due to the 'if' above tags: ${{ steps.meta.outputs.tags }} cache-from: type=registry,ref=${{ env.IMAGE }}:testing # Use testing cache cache-to: type=inline + # The duplicate devmeta block was here and is now removed by not including it in the REPLACE section. + # Removed the 'publish' job for GitHub Pages CV deployment release: diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index bf25c74..f2e2755 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,10 +1,10 @@ -# Active Context: Resolved CI Test Failures & Standardized E2E Tests (2025-04-30) +# Active Context: CI Workflow Improvements (2025-04-30) ## Current Focus -Completing the task related to switching from submodules to system packages for testing and resolving subsequent CI issues. +Finalizing CI workflow improvements related to Docker image tagging, push logic, and caching. -## Problem Identified & Resolved +## Problem Identified & Resolved (Previous) 1. **Initial Problem:** CI tests failed (`bats: not found`) because `actions/checkout@v4` wasn't initializing submodules. 2. **Intermediate Fix:** Added explicit `git submodule update --init --recursive` to CI. @@ -47,26 +47,17 @@ Completing the task related to switching from submodules to system packages for ## Recent Actions (This Session) -- Diagnosed the SSH key failure during `docker build` (previous step). -- Switched test dependencies to system packages and updated related files (previous step). -- Diagnosed the subsequent CI failures (`chmod` error, missing symlinks, missing host Bats helpers). -- Planned the fixes for the CI issues. -- Modified `.devcontainer/Dockerfile.ubuntu` to add Typst package symlink creation (and subsequently removed duplicated commands). -- Modified `tests/unit/build_sh.bats` to remove `chmod`. -- Modified `.github/workflows/ci.yml` to install Bats helpers on the host runner. -- Diagnosed the PDF write permission error (`os error 13`) in CI unit tests. -- Modified `tests/unit/build_sh.bats` again to write test output to `$BATS_TMPDIR/test_output/` instead of the CWD. -- Diagnosed E2E test failures (Pandoc temp file permission, missing templates/filters). -- Converted `tests/test_e2e.sh` to `tests/test_e2e.bats`, incorporating `cd $BATS_TMPDIR` fix and correcting assertion syntax. -- Updated `justfile` to use `tests/test_e2e.bats`. -- Modified `build.sh` to use absolute paths for templates/filters (later reverted). -- Diagnosed Docker test failures caused by absolute paths in `build.sh`. -- Reverted `build.sh` to use relative paths for templates/filters. -- Modified `.devcontainer/Dockerfile.ubuntu` to add symlinks for Pandoc templates/filters to system locations. -- User confirmed tests pass locally after rebuilding devcontainer. -- User deleted `tests/test_e2e.sh`. -- Diagnosed and fixed the `test-docker` CI failure by parameterizing the image tag. -- Diagnosed and fixed the "Build Example PDFs" CI failure by correcting the arguments passed to the Docker entrypoint. +- Diagnosed CI issues: missing tags on PR builds (`ERROR: tag is needed...`), incorrect push logic for production image on PRs, lack of devcontainer caching. +- Planned CI fixes: Add `type=ref,event=pr` tag for production image metadata, make production image push conditional on `github.event_name == 'push'`, implement devcontainer caching using GHCR. +- Modified `.github/workflows/ci.yml`: + * Added `DEV_IMAGE_NAME` environment variable. + * Added unconditional login step for GHCR. + * Added `type=ref,event=pr` to production image metadata (`id: meta`). + * Added metadata step for devcontainer image (`id: devmeta`). + * Replaced manual devcontainer build with `docker/build-push-action` using GHCR for caching (`id: build_devcontainer`). + * Added step to pull cached devcontainer image before internal tests. + * Updated internal test step to use the pulled GHCR image. + * Made the final production image push step (`Build and push`) conditional: `if: success() && (github.event_name == 'push')`. - Updated this `activeContext.md`. ## Decisions & Notes @@ -77,10 +68,12 @@ Completing the task related to switching from submodules to system packages for - Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. - Using an environment variable (`DOCKER_IMAGE_TAG`) allows the Docker usage tests (`tests/docker.bats`) to work correctly in both local (defaulting to `typst-cv:latest`) and CI environments (using the specific testing tag). - When using `docker run` with an `ENTRYPOINT`, the command specified after the image name is passed as arguments *to* the entrypoint script. +- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). +- **Production Image Push:** Only push the final tagged production image to Docker Hub on `push` events (to `main` or tags), not on `pull_request` events. Add `type=ref,event=pr` tag to metadata to ensure a tag always exists for PR builds, even though it won't be pushed. ## Immediate Next Steps - Update `progress.md`. - **User Action:** Commit the changes (including `.github/workflows/ci.yml` and Memory Bank files). -- **User Action:** Trigger the CI workflow and verify all tests pass, including the "Build Example PDFs" step. +- **User Action:** Trigger the CI workflow and verify all tests pass, including the devcontainer caching and conditional push logic. - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 04be157..e339f83 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -10,12 +10,21 @@ - Unit tests (`bats`) for `build.sh` logic exist and pass (`tests/unit/build_sh.bats`). - Filter tests (`bats`) comparing Pandoc output against snapshots exist and pass (`tests/filter/filters.bats`). - E2E smoke tests converted to Bats (`tests/test_e2e.bats`), verifying PDF generation and basic content. Uses `cd $BATS_TMPDIR` workaround for Pandoc temp file permissions. - - Docker usage tests (`bats`) verify production container interaction (`tests/docker.bats`). CI host runner correctly installs helpers. **CI execution fixed** by passing the correct image tag via `DOCKER_IMAGE_TAG` environment variable. + - Docker usage tests (`bats`) verify production container interaction (`tests/docker.bats`). CI host runner correctly installs helpers. CI execution fixed by passing the correct image tag via `DOCKER_IMAGE_TAG` environment variable. - `justfile` provides convenient local test execution commands (`just test`, `just test-unit`, `just test-filter`, `just test-e2e`) using system-installed `bats`. - **Docker:** - Primary production image (`Dockerfile`) is Alpine-based, self-contained, uses multi-stage builds, includes pinned Typst v0.12.0 and necessary packages/fonts. Build context is cleaned via `.dockerignore`. Files (`*.lua`, `*.typ`) copied to standard Pandoc/Typst locations. - - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats. Includes built-in symlinks for the local Typst package (`style.typ`, `typst.toml`) *and* for Pandoc templates/filters (`*.typ`, `*.lua`) into standard system locations, aligning it with the production environment. -- **CI:** GitHub Actions workflow lints code, builds the production Docker image, builds the devcontainer image, runs internal tests (unit, filter, e2e) in the devcontainer image (all known permission/path issues resolved), runs Docker usage tests against the production image (fixed missing host Bats helpers), builds example PDFs using the production container (**fixed entrypoint argument issue**), pushes images, and handles releases. Submodule handling removed. + - Devcontainer (`.devcontainer/Dockerfile.ubuntu`) is Ubuntu-based with dev tools, Docker-in-Docker, system-installed Bats. Includes built-in symlinks for Pandoc templates/filters (`*.typ`, `*.lua`) into standard system locations, aligning it with the production environment. **CI now builds and caches this image to GHCR.** +- **CI:** GitHub Actions workflow: + - Lints code (`pre-commit`). + - Builds production Docker image (`Dockerfile`) and loads it locally for testing (`${{ env.IMAGE_TAG_TESTING }}`). + - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** + - Runs internal tests (unit, filter, e2e) **inside the pulled GHCR devcontainer image**. All known permission/path issues resolved. + - Runs Docker usage tests against the local production testing image (`${{ env.IMAGE_TAG_TESTING }}`). Fixed missing host Bats helpers and image tag mismatch. + - Builds example PDFs using the local production testing image. Fixed entrypoint argument issue. + - **Pushes final tagged production image to Docker Hub *only* on `push` events (main/tags).** Added `type=ref,event=pr` tag to metadata to prevent errors. + - Handles GitHub Releases. + - Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -38,9 +47,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed:** Resolved SSH key errors during production build by adding `.dockerignore`. Removed submodule handling steps. Updated test files and `justfile` to work with system Bats. Resolved subsequent CI failures (`chmod` error, missing Typst package symlinks, PDF write permission error, missing host Bats helpers, Pandoc temp file permissions, Pandoc template/filter path issues, **Docker usage test image tag mismatch**, **Build Example PDFs entrypoint argument mismatch**) by modifying tests, Dockerfiles, build script, and CI workflow as needed. Standardized E2E tests to Bats. +- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for production image builds during PRs. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable, all known CI issues are addressed (including `test-docker` and example builds), tests pass locally. Ready for user to commit changes and trigger CI workflow for final confirmation. +- **Ready for Verification:** Project is stable. Ready for user to commit changes and trigger CI workflow to verify caching and conditional push logic. ## Known Issues From c1baee8c6e690fac97d701c1be05de46bd26338c Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 22:53:13 +0000 Subject: [PATCH 56/63] Add PR event trigger to Devcontainer image caching job --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe0aad9..2e74c87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,7 @@ jobs: tags: | type=ref,event=branch type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=pr - name: Build and Cache Devcontainer Image id: build_devcontainer From 141950b4cc5ed011582df0cd593f7681f27e0067 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Wed, 30 Apr 2025 23:18:16 +0000 Subject: [PATCH 57/63] Ensure lowercase repository name in Devcontainer image name --- .github/workflows/ci.yml | 6 +++++- memory-bank/activeContext.md | 6 ++++-- memory-bank/progress.md | 7 ++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e74c87..d847fa3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: OUTPUT_ARTIFACTS_NAME: pdf EXAMPLE_ARTIFACTS_NAME: examples TEST_DIR: tests # Directory containing test scripts and fixtures - DEV_IMAGE_NAME: ghcr.io/${{ github.repository }}/devcontainer + # DEV_IMAGE_NAME will be set dynamically in the job jobs: pre_commit: @@ -79,6 +79,10 @@ jobs: # Submodules are no longer used for testing dependencies # submodules: recursive # Keep commented out as reference if needed + - name: Set Devcontainer Image Name Env Var + run: echo "DEV_IMAGE_NAME=ghcr.io/$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')/devcontainer" >> "$GITHUB_ENV" + shell: bash + - name: Devcontainer Docker meta # For devcontainer cache image id: devmeta # Ensure this ID is unique uses: docker/metadata-action@v5 diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index f2e2755..75b60d5 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -50,7 +50,6 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, - Diagnosed CI issues: missing tags on PR builds (`ERROR: tag is needed...`), incorrect push logic for production image on PRs, lack of devcontainer caching. - Planned CI fixes: Add `type=ref,event=pr` tag for production image metadata, make production image push conditional on `github.event_name == 'push'`, implement devcontainer caching using GHCR. - Modified `.github/workflows/ci.yml`: - * Added `DEV_IMAGE_NAME` environment variable. * Added unconditional login step for GHCR. * Added `type=ref,event=pr` to production image metadata (`id: meta`). * Added metadata step for devcontainer image (`id: devmeta`). @@ -58,6 +57,9 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, * Added step to pull cached devcontainer image before internal tests. * Updated internal test step to use the pulled GHCR image. * Made the final production image push step (`Build and push`) conditional: `if: success() && (github.event_name == 'push')`. + * Corrected step order for `devmeta`. + * Added `type=ref,event=pr` to devcontainer image metadata (`id: devmeta`). + * Corrected GHCR image name definition: removed `DEV_IMAGE_NAME` from top-level `env`, added a `run` step within the `docker` job to set `DEV_IMAGE_NAME` dynamically using `echo ... | tr '[:upper:]' '[:lower:]'` and `$GITHUB_ENV`. - Updated this `activeContext.md`. ## Decisions & Notes @@ -68,7 +70,7 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, - Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. - Using an environment variable (`DOCKER_IMAGE_TAG`) allows the Docker usage tests (`tests/docker.bats`) to work correctly in both local (defaulting to `typst-cv:latest`) and CI environments (using the specific testing tag). - When using `docker run` with an `ENTRYPOINT`, the command specified after the image name is passed as arguments *to* the entrypoint script. -- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). +- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`, ensuring lowercase) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). Set the `DEV_IMAGE_NAME` env var dynamically within the job using a `run` step and `$GITHUB_ENV`. - **Production Image Push:** Only push the final tagged production image to Docker Hub on `push` events (to `main` or tags), not on `pull_request` events. Add `type=ref,event=pr` tag to metadata to ensure a tag always exists for PR builds, even though it won't be pushed. ## Immediate Next Steps diff --git a/memory-bank/progress.md b/memory-bank/progress.md index e339f83..1fd79cb 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -18,11 +18,12 @@ - **CI:** GitHub Actions workflow: - Lints code (`pre-commit`). - Builds production Docker image (`Dockerfile`) and loads it locally for testing (`${{ env.IMAGE_TAG_TESTING }}`). - - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** + - **Sets lowercase GHCR image name (`DEV_IMAGE_NAME`) dynamically.** + - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** Includes PR tagging. - Runs internal tests (unit, filter, e2e) **inside the pulled GHCR devcontainer image**. All known permission/path issues resolved. - Runs Docker usage tests against the local production testing image (`${{ env.IMAGE_TAG_TESTING }}`). Fixed missing host Bats helpers and image tag mismatch. - Builds example PDFs using the local production testing image. Fixed entrypoint argument issue. - - **Pushes final tagged production image to Docker Hub *only* on `push` events (main/tags).** Added `type=ref,event=pr` tag to metadata to prevent errors. + - **Pushes final tagged production image to Docker Hub *only* on `push` events (main/tags).** Includes PR tagging. - Handles GitHub Releases. - Submodule handling removed. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. @@ -47,7 +48,7 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for production image builds during PRs. +- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. - **Ready for Verification:** Project is stable. Ready for user to commit changes and trigger CI workflow to verify caching and conditional push logic. From 98a9439c7de09fd2c869cfb18bb60357ff60af1b Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 1 May 2025 10:12:35 +0000 Subject: [PATCH 58/63] Add permissions for Docker job to enable GHCR package push --- .github/workflows/ci.yml | 3 +++ memory-bank/activeContext.md | 6 ++++-- memory-bank/progress.md | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d847fa3..1fe0a98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,9 @@ jobs: name: Build, test, and push Docker image needs: pre_commit # Doesn't need source artifact anymore, tests are in repo runs-on: ubuntu-latest + permissions: + contents: read # Needed for checkout + packages: write # Needed to push to GHCR steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 75b60d5..bf462ea 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -1,4 +1,4 @@ -# Active Context: CI Workflow Improvements (2025-04-30) +# Active Context: CI Workflow Improvements (2025-05-01) ## Current Focus @@ -60,6 +60,8 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, * Corrected step order for `devmeta`. * Added `type=ref,event=pr` to devcontainer image metadata (`id: devmeta`). * Corrected GHCR image name definition: removed `DEV_IMAGE_NAME` from top-level `env`, added a `run` step within the `docker` job to set `DEV_IMAGE_NAME` dynamically using `echo ... | tr '[:upper:]' '[:lower:]'` and `$GITHUB_ENV`. + * Resolved `shellcheck` warning by double-quoting `$GITHUB_ENV`. + * Added `permissions: packages: write` to the `docker` job to allow pushing to GHCR. - Updated this `activeContext.md`. ## Decisions & Notes @@ -70,7 +72,7 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, - Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. - Using an environment variable (`DOCKER_IMAGE_TAG`) allows the Docker usage tests (`tests/docker.bats`) to work correctly in both local (defaulting to `typst-cv:latest`) and CI environments (using the specific testing tag). - When using `docker run` with an `ENTRYPOINT`, the command specified after the image name is passed as arguments *to* the entrypoint script. -- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`, ensuring lowercase) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). Set the `DEV_IMAGE_NAME` env var dynamically within the job using a `run` step and `$GITHUB_ENV`. +- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`, ensuring lowercase) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). Set the `DEV_IMAGE_NAME` env var dynamically within the job using a `run` step and `$GITHUB_ENV`. Requires `permissions: packages: write` on the job. - **Production Image Push:** Only push the final tagged production image to Docker Hub on `push` events (to `main` or tags), not on `pull_request` events. Add `type=ref,event=pr` tag to metadata to ensure a tag always exists for PR builds, even though it won't be pushed. ## Immediate Next Steps diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 1fd79cb..e26eb66 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -1,4 +1,4 @@ -# Progress: Typst CV/Letter Generator (As of 2025-04-30) +# Progress: Typst CV/Letter Generator (As of 2025-05-01) ## What Works (Current State) @@ -19,7 +19,7 @@ - Lints code (`pre-commit`). - Builds production Docker image (`Dockerfile`) and loads it locally for testing (`${{ env.IMAGE_TAG_TESTING }}`). - **Sets lowercase GHCR image name (`DEV_IMAGE_NAME`) dynamically.** - - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** Includes PR tagging. + - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** Includes PR tagging. **Added `permissions: packages: write` to job.** - Runs internal tests (unit, filter, e2e) **inside the pulled GHCR devcontainer image**. All known permission/path issues resolved. - Runs Docker usage tests against the local production testing image (`${{ env.IMAGE_TAG_TESTING }}`). Fixed missing host Bats helpers and image tag mismatch. - Builds example PDFs using the local production testing image. Fixed entrypoint argument issue. @@ -48,7 +48,7 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. +- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. - **Ready for Verification:** Project is stable. Ready for user to commit changes and trigger CI workflow to verify caching and conditional push logic. From 980e0302fdb3f210840e898d355b6583c5a8d422 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 1 May 2025 11:31:42 +0000 Subject: [PATCH 59/63] Remove redundant source job and docker pull step from CI workflow --- .github/workflows/ci.yml | 21 +++------------------ memory-bank/activeContext.md | 5 +++-- memory-bank/progress.md | 7 ++++--- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe0a98..7770202 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,25 +29,11 @@ jobs: - uses: pre-commit-ci/lite-action@v1.0.3 if: always() - source: - name: Extract example source files - needs: pre_commit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Upload source files - uses: actions/upload-artifact@v4 - with: - name: ${{ env.EXAMPLE_ARTIFACTS_NAME }} - path: | - ${{ env.TEST_DIR }}/fixtures/example-*.md - ${{ env.TEST_DIR }}/fixtures/placeholder-photo.png - if-no-files-found: error + # The 'source' job previously here is removed as it's no longer needed. docker: name: Build, test, and push Docker image - needs: pre_commit # Doesn't need source artifact anymore, tests are in repo + needs: pre_commit # Now only depends on pre_commit runs-on: ubuntu-latest permissions: contents: read # Needed for checkout @@ -110,8 +96,7 @@ jobs: - name: Install Bats, Helpers, and File command (for Docker usage tests on host) run: sudo apt-get update && sudo apt-get install -y bats bats-support bats-assert file - - name: Pull Cached Devcontainer Image - run: docker pull ${{ env.DEV_IMAGE_NAME }}:latest + # The explicit 'docker pull' step previously here is removed as it's redundant. - name: Run internal tests (unit, filter, e2e) inside Devcontainer Image run: | diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index bf462ea..5f5695c 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -54,14 +54,14 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, * Added `type=ref,event=pr` to production image metadata (`id: meta`). * Added metadata step for devcontainer image (`id: devmeta`). * Replaced manual devcontainer build with `docker/build-push-action` using GHCR for caching (`id: build_devcontainer`). - * Added step to pull cached devcontainer image before internal tests. - * Updated internal test step to use the pulled GHCR image. * Made the final production image push step (`Build and push`) conditional: `if: success() && (github.event_name == 'push')`. * Corrected step order for `devmeta`. * Added `type=ref,event=pr` to devcontainer image metadata (`id: devmeta`). * Corrected GHCR image name definition: removed `DEV_IMAGE_NAME` from top-level `env`, added a `run` step within the `docker` job to set `DEV_IMAGE_NAME` dynamically using `echo ... | tr '[:upper:]' '[:lower:]'` and `$GITHUB_ENV`. * Resolved `shellcheck` warning by double-quoting `$GITHUB_ENV`. * Added `permissions: packages: write` to the `docker` job to allow pushing to GHCR. + * Removed redundant `source` job. + * Removed redundant `docker pull ${{ env.DEV_IMAGE_NAME }}:latest` step. - Updated this `activeContext.md`. ## Decisions & Notes @@ -74,6 +74,7 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, - When using `docker run` with an `ENTRYPOINT`, the command specified after the image name is passed as arguments *to* the entrypoint script. - **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`, ensuring lowercase) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). Set the `DEV_IMAGE_NAME` env var dynamically within the job using a `run` step and `$GITHUB_ENV`. Requires `permissions: packages: write` on the job. - **Production Image Push:** Only push the final tagged production image to Docker Hub on `push` events (to `main` or tags), not on `pull_request` events. Add `type=ref,event=pr` tag to metadata to ensure a tag always exists for PR builds, even though it won't be pushed. +- **Redundant Steps Removed:** The `source` job and the explicit `docker pull` for the devcontainer image were removed as they were unnecessary. ## Immediate Next Steps diff --git a/memory-bank/progress.md b/memory-bank/progress.md index e26eb66..79973be 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -19,13 +19,14 @@ - Lints code (`pre-commit`). - Builds production Docker image (`Dockerfile`) and loads it locally for testing (`${{ env.IMAGE_TAG_TESTING }}`). - **Sets lowercase GHCR image name (`DEV_IMAGE_NAME`) dynamically.** - - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** Includes PR tagging. **Added `permissions: packages: write` to job.** - - Runs internal tests (unit, filter, e2e) **inside the pulled GHCR devcontainer image**. All known permission/path issues resolved. + - **Builds and caches devcontainer image to GHCR (`${{ env.DEV_IMAGE_NAME }}:latest`) on every run.** Includes PR tagging. Added `permissions: packages: write` to job. + - Runs internal tests (unit, filter, e2e) **inside the GHCR devcontainer image** (implicit pull via `docker run`). All known permission/path issues resolved. - Runs Docker usage tests against the local production testing image (`${{ env.IMAGE_TAG_TESTING }}`). Fixed missing host Bats helpers and image tag mismatch. - Builds example PDFs using the local production testing image. Fixed entrypoint argument issue. - **Pushes final tagged production image to Docker Hub *only* on `push` events (main/tags).** Includes PR tagging. - Handles GitHub Releases. - Submodule handling removed. + - **Redundant `source` job and explicit `docker pull` step removed.** - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -48,7 +49,7 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. +- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. - **Ready for Verification:** Project is stable. Ready for user to commit changes and trigger CI workflow to verify caching and conditional push logic. From 3e485043ea71c175beb023af6e2c46610b2179a7 Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 1 May 2025 12:20:26 +0000 Subject: [PATCH 60/63] Fix devcontainer image tagging for PR builds by ensuring the :latest tag is always pushed to GHCR --- .github/workflows/ci.yml | 2 +- memory-bank/activeContext.md | 3 +++ memory-bank/progress.md | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7770202..3c49b6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: images: ${{ env.DEV_IMAGE_NAME }} tags: | type=ref,event=branch - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest type=ref,event=pr - name: Build and Cache Devcontainer Image diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 5f5695c..0f3f42d 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -44,6 +44,9 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, 19. **Root Cause (CI - Example Build):** The `docker run` commands in the CI workflow incorrectly included `build.sh` as part of the command being passed to the container. Since the container has `ENTRYPOINT ["build.sh"]`, the script name was passed as the first argument to the entrypoint script itself, leading to the argument parsing error. 20. **Solution Implemented (CI - Example Build):** * Modified `.github/workflows/ci.yml` to remove the redundant `build.sh` from the `docker run` commands in the "Build Example PDFs" step, ensuring only the intended arguments are passed to the entrypoint. +21. **New Problem (CI - Devcontainer Tests):** The "Run internal tests" step failed with `docker: manifest unknown` for `ghcr.io/.../devcontainer:latest` during PR builds. +22. **Root Cause (CI - Devcontainer Tests):** The `devmeta` step in `.github/workflows/ci.yml` only added the `:latest` tag for the devcontainer image on the default branch (`main`), not on PRs. The subsequent test step explicitly tried to pull `:latest`, which didn't exist for PR builds. +23. **Solution Implemented (CI - Devcontainer Tests):** Modified the `devmeta` step in `.github/workflows/ci.yml` to remove the `enable={{is_default_branch}}` condition, ensuring the `:latest` tag is always generated and pushed for the devcontainer image, making it available for the test step during PR builds. ## Recent Actions (This Session) diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 79973be..21962c5 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -49,9 +49,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. +- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. **Fixed "manifest unknown" error for devcontainer tests during PR builds by ensuring the `:latest` tag is always pushed to GHCR.** - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable. Ready for user to commit changes and trigger CI workflow to verify caching and conditional push logic. +- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger CI workflow to verify the final fixes, caching, and conditional push logic. ## Known Issues From 7dc1a5ae37b7fdf4f93be26b67fbed12055a864a Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 1 May 2025 13:20:57 +0000 Subject: [PATCH 61/63] Refactor devcontainer image caching to use dedicated ':buildcache' tag for cache management, ensuring consistent updates to the ':latest' tag during PR builds. --- .github/workflows/ci.yml | 4 ++-- memory-bank/activeContext.md | 6 +++++- memory-bank/progress.md | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c49b6c..baf78c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,8 +90,8 @@ jobs: file: .devcontainer/Dockerfile.ubuntu tags: ${{ steps.devmeta.outputs.tags }} push: true # Always push cache to GHCR - cache-from: type=registry,ref=${{ env.DEV_IMAGE_NAME }}:latest - cache-to: type=registry,ref=${{ env.DEV_IMAGE_NAME }}:latest,mode=max # Cache to latest tag + cache-from: type=registry,ref=${{ env.DEV_IMAGE_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.DEV_IMAGE_NAME }}:buildcache,mode=max - name: Install Bats, Helpers, and File command (for Docker usage tests on host) run: sudo apt-get update && sudo apt-get install -y bats bats-support bats-assert file diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 0f3f42d..31f2aa2 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -47,6 +47,9 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, 21. **New Problem (CI - Devcontainer Tests):** The "Run internal tests" step failed with `docker: manifest unknown` for `ghcr.io/.../devcontainer:latest` during PR builds. 22. **Root Cause (CI - Devcontainer Tests):** The `devmeta` step in `.github/workflows/ci.yml` only added the `:latest` tag for the devcontainer image on the default branch (`main`), not on PRs. The subsequent test step explicitly tried to pull `:latest`, which didn't exist for PR builds. 23. **Solution Implemented (CI - Devcontainer Tests):** Modified the `devmeta` step in `.github/workflows/ci.yml` to remove the `enable={{is_default_branch}}` condition, ensuring the `:latest` tag is always generated and pushed for the devcontainer image, making it available for the test step during PR builds. +24. **New Problem (CI - Devcontainer Tests - Persistent):** Despite fix #23, the "Run internal tests" step *still* failed with `docker: manifest unknown` for `:latest` locally and in CI, even though logs showed `:latest` was tagged. The `:latest` tag on GHCR was not updating consistently. +25. **Root Cause (CI - Devcontainer Tests - Persistent):** Using the same `:latest` tag for both the image tag (`tags:`) and the cache reference (`cache-from`/`cache-to:`) in `docker/build-push-action` likely interfered with the proper updating of the `:latest` image manifest. +26. **Solution Implemented (CI - Devcontainer Tests - Persistent):** Modified the `Build and Cache Devcontainer Image` step in `.github/workflows/ci.yml` to use a dedicated cache tag (`:buildcache`) for `cache-from` and `cache-to`, separating cache management from the primary `:latest` image tag. ## Recent Actions (This Session) @@ -65,6 +68,7 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, * Added `permissions: packages: write` to the `docker` job to allow pushing to GHCR. * Removed redundant `source` job. * Removed redundant `docker pull ${{ env.DEV_IMAGE_NAME }}:latest` step. + * Modified `Build and Cache Devcontainer Image` step to use `:buildcache` for `cache-from`/`cache-to` instead of `:latest`. - Updated this `activeContext.md`. ## Decisions & Notes @@ -75,7 +79,7 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, - Running tests that write output from within `$BATS_TMPDIR` avoids container volume mount permission issues. - Using an environment variable (`DOCKER_IMAGE_TAG`) allows the Docker usage tests (`tests/docker.bats`) to work correctly in both local (defaulting to `typst-cv:latest`) and CI environments (using the specific testing tag). - When using `docker run` with an `ENTRYPOINT`, the command specified after the image name is passed as arguments *to* the entrypoint script. -- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`, ensuring lowercase) for caching the devcontainer image. Cache to/from the `:latest` tag. Push cache on every run (PRs and pushes). Set the `DEV_IMAGE_NAME` env var dynamically within the job using a `run` step and `$GITHUB_ENV`. Requires `permissions: packages: write` on the job. +- **Devcontainer Caching:** Use GHCR (`ghcr.io/${{ github.repository }}/devcontainer`, ensuring lowercase) for caching the devcontainer image. Use a dedicated tag (`:buildcache`) for `cache-from` and `cache-to` refs. Push cache on every run (PRs and pushes). Set the `DEV_IMAGE_NAME` env var dynamically within the job using a `run` step and `$GITHUB_ENV`. Requires `permissions: packages: write` on the job. The image itself should still be tagged with `:latest` (along with other dynamic tags like `pr-X`). - **Production Image Push:** Only push the final tagged production image to Docker Hub on `push` events (to `main` or tags), not on `pull_request` events. Add `type=ref,event=pr` tag to metadata to ensure a tag always exists for PR builds, even though it won't be pushed. - **Redundant Steps Removed:** The `source` job and the explicit `docker pull` for the devcontainer image were removed as they were unnecessary. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 21962c5..ee15dea 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -49,9 +49,9 @@ - **Docker/CI Refactoring Complete:** Production Docker image switched to Alpine, CI updated to use it. - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. -- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. **Fixed "manifest unknown" error for devcontainer tests during PR builds by ensuring the `:latest` tag is always pushed to GHCR.** +- **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions, **and dedicated `:buildcache` tag**) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. **Fixed persistent "manifest unknown" error for devcontainer tests by separating the cache tag (`:buildcache`) from the image tag (`:latest`) in the `docker/build-push-action` step.** - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger CI workflow to verify the final fixes, caching, and conditional push logic. +- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger CI workflow to verify the final fixes, caching (using `:buildcache`), and conditional push logic. ## Known Issues From 5a5eabfa1ff01d7cf91cda125059d329cfd5f83e Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 1 May 2025 15:24:05 +0000 Subject: [PATCH 62/63] Add CI/CD enhancements: integrate CodeQL analysis, Docker image scanning with Trivy, and Dependabot configuration --- .github/dependabot.yml | 34 +++++++++++++++++ .github/workflows/ci.yml | 25 ++++++++++++- .github/workflows/codeql-analysis.yml | 53 +++++++++++++++++++++++++++ memory-bank/activeContext.md | 11 +++++- memory-bank/progress.md | 6 ++- 5 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..26da543 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,34 @@ +--- +# Basic Dependabot configuration file + +version: 2 +updates: + # Maintain dependencies for Docker + - package-ecosystem: "docker" + directory: "/" # Location of Dockerfile + schedule: + interval: "weekly" + day: "sunday" # Check on Sundays + # Add reviewers, assignees, labels as needed + # reviewers: + # - "octocat" + # assignees: + # - "octocat" + # labels: + # - "dependencies" + # - "docker" + + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" # Location of workflow files + schedule: + interval: "weekly" + day: "sunday" # Check on Sundays + # Add reviewers, assignees, labels as needed + # reviewers: + # - "octocat" + # assignees: + # - "octocat" + # labels: + # - "dependencies" + # - "github_actions" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baf78c8..f18ff19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,11 +166,34 @@ jobs: # The duplicate devmeta block was here and is now removed by not including it in the REPLACE section. + scan-image: + name: Scan Docker image with Trivy + needs: docker # Run after the image is built and tested + runs-on: ubuntu-latest + permissions: + contents: read # Needed for checkout + security-events: write # Needed for reporting findings (optional) + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master # Use master for latest features/fixes + with: + image-ref: ${{ env.IMAGE_TAG_TESTING }} # Scan the testing image + format: 'table' + exit-code: '1' # Fail the workflow if vulnerabilities are found + ignore-unfixed: true # Don't fail on vulnerabilities without known fixes + vuln-type: 'os,library' # Scan OS packages and language-specific packages + severity: 'HIGH,CRITICAL' # Fail only on HIGH or CRITICAL severity vulnerabilities + # Optional: Upload results to GitHub Security tab + # github-pat: ${{ secrets.GITHUB_TOKEN }} # Requires security-events: write permission + # Removed the 'publish' job for GitHub Pages CV deployment release: name: Create a GitHub release - needs: docker + needs: scan-image # Now depends on the scan job passing runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/tags/') }} permissions: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..cacc100 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,53 @@ +--- +name: "CodeQL Analysis" + +on: + push: + branches: [main] + pull_request: + # The branches below must be a subset of the branches above + branches: [main] + schedule: + - cron: '30 1 * * 0' # Run weekly on Sunday at 01:30 UTC + +permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: ['bash', 'lua'] + # CodeQL supports bash and lua. + # Learn more about CodeQL language support: https://docs.github.com/en/code-security/code-scanning/introduction-to-code-scanning/about-code-scanning#supported-languages-and-frameworks + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, Swift, Kotlin). + # Since Bash and Lua are interpreted, this step might be minimal but is kept for standard practice. + # If you had compiled languages, you would typically add your build steps here. + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index 31f2aa2..dbe2322 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -83,9 +83,16 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, - **Production Image Push:** Only push the final tagged production image to Docker Hub on `push` events (to `main` or tags), not on `pull_request` events. Add `type=ref,event=pr` tag to metadata to ensure a tag always exists for PR builds, even though it won't be pushed. - **Redundant Steps Removed:** The `source` job and the explicit `docker pull` for the devcontainer image were removed as they were unnecessary. +## Recent Actions (This Session Continued) + +- **Added CodeQL:** Created `.github/workflows/codeql-analysis.yml` to perform static analysis on Bash and Lua code on push/PR/schedule. +- **Added Docker Image Scanning:** Added a `scan-image` job to `.github/workflows/ci.yml` using `aquasecurity/trivy-action` to scan the `${{ env.IMAGE_TAG_TESTING }}` image for HIGH/CRITICAL vulnerabilities after the `docker` job and before the `release` job. Updated `release` job dependency. +- **Added Dependabot:** Created `.github/dependabot.yml` to configure weekly checks for updates to the base Docker image and GitHub Actions used in workflows. +- Updated this `activeContext.md`. + ## Immediate Next Steps - Update `progress.md`. -- **User Action:** Commit the changes (including `.github/workflows/ci.yml` and Memory Bank files). -- **User Action:** Trigger the CI workflow and verify all tests pass, including the devcontainer caching and conditional push logic. +- **User Action:** Commit the changes (including `.github/workflows/ci.yml`, `.github/workflows/codeql-analysis.yml`, `.github/dependabot.yml`, and Memory Bank files). +- **User Action:** Trigger the CI workflows (`ci.yml`, `codeql-analysis.yml`) and verify they pass. Check Dependabot configuration in repository settings. - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index ee15dea..9009839 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -27,6 +27,9 @@ - Handles GitHub Releases. - Submodule handling removed. - **Redundant `source` job and explicit `docker pull` step removed.** + - **Added CodeQL:** New workflow (`.github/workflows/codeql-analysis.yml`) performs static analysis for Bash and Lua. + - **Added Docker Image Scanning:** New `scan-image` job in `ci.yml` uses Trivy to check the testing image for HIGH/CRITICAL vulnerabilities before release. + - **Added Dependabot:** New configuration (`.github/dependabot.yml`) schedules weekly checks for Dockerfile and GitHub Actions updates. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. - **Test Dependencies:** Bats, Bats-Support, Bats-Assert are installed via system package manager (`apt`) in the devcontainer and CI test environments, replacing Git submodules. Test files updated to use `bats_load_library`. @@ -50,8 +53,9 @@ - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. - **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions, **and dedicated `:buildcache` tag**) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. **Fixed persistent "manifest unknown" error for devcontainer tests by separating the cache tag (`:buildcache`) from the image tag (`:latest`) in the `docker/build-push-action` step.** +- **Added Security Checks:** Integrated CodeQL static analysis, Trivy Docker image vulnerability scanning, and Dependabot dependency updates into the CI/CD process. - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger CI workflow to verify the final fixes, caching (using `:buildcache`), and conditional push logic. +- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger CI workflows (`ci.yml`, `codeql-analysis.yml`) to verify the final fixes, caching, conditional push logic, and new security scans. User should also check Dependabot configuration in repository settings. ## Known Issues From 0108d138a4c812de5eb38aa90dd8b3b80d42dc1d Mon Sep 17 00:00:00 2001 From: Aleksandr Kadykov Date: Thu, 1 May 2025 16:04:45 +0000 Subject: [PATCH 63/63] Remove CodeQL analysis workflow due to unsupported languages and errors --- .github/workflows/codeql-analysis.yml | 53 --------------------------- memory-bank/activeContext.md | 6 +-- memory-bank/progress.md | 5 +-- 3 files changed, 5 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index cacc100..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "CodeQL Analysis" - -on: - push: - branches: [main] - pull_request: - # The branches below must be a subset of the branches above - branches: [main] - schedule: - - cron: '30 1 * * 0' # Run weekly on Sunday at 01:30 UTC - -permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - language: ['bash', 'lua'] - # CodeQL supports bash and lua. - # Learn more about CodeQL language support: https://docs.github.com/en/code-security/code-scanning/introduction-to-code-scanning/about-code-scanning#supported-languages-and-frameworks - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v3 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, Swift, Kotlin). - # Since Bash and Lua are interpreted, this step might be minimal but is kept for standard practice. - # If you had compiled languages, you would typically add your build steps here. - - name: Autobuild - uses: github/codeql-action/autobuild@v3 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 - with: - category: "/language:${{matrix.language}}" diff --git a/memory-bank/activeContext.md b/memory-bank/activeContext.md index dbe2322..4a7900f 100644 --- a/memory-bank/activeContext.md +++ b/memory-bank/activeContext.md @@ -85,7 +85,7 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, ## Recent Actions (This Session Continued) -- **Added CodeQL:** Created `.github/workflows/codeql-analysis.yml` to perform static analysis on Bash and Lua code on push/PR/schedule. +- **Attempted CodeQL:** Created `.github/workflows/codeql-analysis.yml` but removed it after discovering Lua is unsupported and Bash support caused errors (`Did not recognize the following languages: bash`). - **Added Docker Image Scanning:** Added a `scan-image` job to `.github/workflows/ci.yml` using `aquasecurity/trivy-action` to scan the `${{ env.IMAGE_TAG_TESTING }}` image for HIGH/CRITICAL vulnerabilities after the `docker` job and before the `release` job. Updated `release` job dependency. - **Added Dependabot:** Created `.github/dependabot.yml` to configure weekly checks for updates to the base Docker image and GitHub Actions used in workflows. - Updated this `activeContext.md`. @@ -93,6 +93,6 @@ Finalizing CI workflow improvements related to Docker image tagging, push logic, ## Immediate Next Steps - Update `progress.md`. -- **User Action:** Commit the changes (including `.github/workflows/ci.yml`, `.github/workflows/codeql-analysis.yml`, `.github/dependabot.yml`, and Memory Bank files). -- **User Action:** Trigger the CI workflows (`ci.yml`, `codeql-analysis.yml`) and verify they pass. Check Dependabot configuration in repository settings. +- **User Action:** Commit the changes (including `.github/workflows/ci.yml`, `.github/dependabot.yml`, and Memory Bank files). +- **User Action:** Trigger the CI workflow (`ci.yml`) and verify it passes. Check Dependabot configuration in repository settings. - Complete the task. diff --git a/memory-bank/progress.md b/memory-bank/progress.md index 9009839..ddeee38 100644 --- a/memory-bank/progress.md +++ b/memory-bank/progress.md @@ -27,7 +27,6 @@ - Handles GitHub Releases. - Submodule handling removed. - **Redundant `source` job and explicit `docker pull` step removed.** - - **Added CodeQL:** New workflow (`.github/workflows/codeql-analysis.yml`) performs static analysis for Bash and Lua. - **Added Docker Image Scanning:** New `scan-image` job in `ci.yml` uses Trivy to check the testing image for HIGH/CRITICAL vulnerabilities before release. - **Added Dependabot:** New configuration (`.github/dependabot.yml`) schedules weekly checks for Dockerfile and GitHub Actions updates. - **Photo Handling:** Refactored to use `{photo="path" photowidth="..."}` attributes for better usability. @@ -53,9 +52,9 @@ - **Devcontainer Switched:** Development environment moved to Ubuntu with Docker-in-Docker. - **Test Dependencies Switched:** Successfully migrated from Git submodules to system packages (`apt`) for Bats testing framework. - **CI Workflow Fixed & Improved:** Resolved previous CI failures (SSH keys, permissions, paths, test dependencies, entrypoint args, etc.). Implemented **devcontainer caching via GHCR** (using dynamically set lowercase image name, correct job permissions, **and dedicated `:buildcache` tag**) and **conditional push logic** for production images (only push on `push` events). Added `pr-X` tagging for both production and devcontainer image builds during PRs. Resolved `shellcheck` warning. Removed redundant `source` job and `docker pull` step. **Fixed persistent "manifest unknown" error for devcontainer tests by separating the cache tag (`:buildcache`) from the image tag (`:latest`) in the `docker/build-push-action` step.** -- **Added Security Checks:** Integrated CodeQL static analysis, Trivy Docker image vulnerability scanning, and Dependabot dependency updates into the CI/CD process. +- **Added Security Checks:** Integrated Trivy Docker image vulnerability scanning and Dependabot dependency updates into the CI/CD process. (CodeQL was attempted but removed due to language support issues). - **Submodule Cleanup:** Removed submodule configuration (`.gitmodules`) and directories (`tests/bats`, `tests/test_helper/*`). Old `tests/test_e2e.sh` deleted by user. -- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger CI workflows (`ci.yml`, `codeql-analysis.yml`) to verify the final fixes, caching, conditional push logic, and new security scans. User should also check Dependabot configuration in repository settings. +- **Ready for Verification:** Project is stable. All known CI issues are resolved. Ready for user to commit changes and trigger the CI workflow (`ci.yml`) to verify the final fixes, caching, conditional push logic, and new security scans. User should also check Dependabot configuration in repository settings. ## Known Issues