From 1b972a994b70368166f9051b6a2347a01d5fe09b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:26:09 +0200 Subject: [PATCH 01/11] ci: bump crate-ci/typos from 1.45.1 to 1.48.0 (#5) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ff752fd..8f3ccb2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Spell check repo - uses: crate-ci/typos@cf5f1c29a8ac336af8568821ec41919923b05a83 # v1.45.1 + uses: crate-ci/typos@bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0 with: config: .config/typos.toml From 43fe903b692e884d81498e5bd64c69a45fd614bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:26:14 +0200 Subject: [PATCH 02/11] ci: bump astral-sh/setup-uv from 8.1.0 to 8.3.2 (#4) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9eebcdf..a98941e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: # Install uv to use git-cliff and rumdl - name: Set up uv - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: enable-cache: true From 0f909ae2a68d8f726f176b710b870c44dcda7b8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:26:18 +0200 Subject: [PATCH 03/11] ci: bump step-security/harden-runner from 2.14.0 to 2.20.0 (#3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/add-to-project.yml | 2 +- .github/workflows/build-website.yml | 2 +- .github/workflows/checks.yml | 6 +++--- .github/workflows/release.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index b7f29fc..d743b3f 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -26,7 +26,7 @@ jobs: # This is a useful security step to check for unexpected outbound calls from the runner, # which could indicate a compromised token or runner. - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/build-website.yml b/.github/workflows/build-website.yml index 81509e9..6ba426c 100644 --- a/.github/workflows/build-website.yml +++ b/.github/workflows/build-website.yml @@ -20,7 +20,7 @@ jobs: # This is a useful security step to check for unexpected outbound calls from the runner, # which could indicate a compromised token or runner. - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8f3ccb2..71c29a2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -11,7 +11,7 @@ jobs: # This is a useful security step to check for unexpected outbound calls from the runner, # which could indicate a compromised token or runner. - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -29,7 +29,7 @@ jobs: # This is a useful security step to check for unexpected outbound calls from the runner, # which could indicate a compromised token or runner. - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit @@ -55,7 +55,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a98941e..01f2c8b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: # This is a useful security step to check for unexpected outbound calls from the runner, # which could indicate a compromised token or runner. - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 with: egress-policy: audit From a1387d6265eddea48930a5fc1c86e8c2c49c4d57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:26:23 +0200 Subject: [PATCH 04/11] ci: bump actions/checkout from 6.0.2 to 7.0.0 (#2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-website.yml | 2 +- .github/workflows/checks.yml | 6 +++--- .github/workflows/release.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-website.yml b/.github/workflows/build-website.yml index 6ba426c..1fa0574 100644 --- a/.github/workflows/build-website.yml +++ b/.github/workflows/build-website.yml @@ -25,7 +25,7 @@ jobs: egress-policy: audit - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@8a96df13519ee81fd526f2dfca5962811136661b # v2.2.0 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 71c29a2..5f6ac2b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,7 +16,7 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Spell check repo uses: crate-ci/typos@bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0 @@ -34,7 +34,7 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@8a96df13519ee81fd526f2dfca5962811136661b # v2.2.0 @@ -60,7 +60,7 @@ jobs: egress-policy: audit - name: "Checkout Repository" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: "Dependency Review" uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01f2c8b..959211e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: private-key: ${{ secrets.UPDATE_VERSION_TOKEN }} - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # Only need the last commit from the repo. fetch-depth: 0 From 2b70f21b645685ec3bf019845ac17fb7dcaca6ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:26:28 +0200 Subject: [PATCH 05/11] ci: bump cocogitto/cocogitto-action from 4.1.0 to 4.2.0 (#1) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 959211e..3a0c4bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - name: Install Cocogitto - uses: cocogitto/cocogitto-action@9a9fe03b31c47444290c0d7f9b1ee1b44ee13f20 # v4.1.0 + uses: cocogitto/cocogitto-action@52d0023b4396897f1815648e553db2ab8d782f3a # v4.2.0 with: command: check From 8f1cd28f7213af492a329e2f771c7bc34d0fdb2c Mon Sep 17 00:00:00 2001 From: "Luke W. Johnston" Date: Sat, 11 Jul 2026 16:43:17 +0200 Subject: [PATCH 06/11] =?UTF-8?q?feat:=20=E2=9C=A8=20add=20build=20process?= =?UTF-8?q?=20chapter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/build.qmd | 187 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 1 deletion(-) diff --git a/docs/build.qmd b/docs/build.qmd index 44b1bee..42ac2e7 100644 --- a/docs/build.qmd +++ b/docs/build.qmd @@ -1 +1,186 @@ -# Build process +# Build process {#sec-build} + +Like other types of packages (e.g. Rust, Python, R), the contents of the Git +repository *build* the final data package, but aren't the data package itself. +The repository contains the source of the package, such as the raw data input +and the code to process the data into the final format. But it isn't the package +itself. We need to first "compile" the package by running the code to take the +raw data input and build the final data package. + +At a high level, the directories and files involved in the build process are +(non-relevant files and directories are removed): + +``` +/ +├── .github/ +│ └── workflows/ +│ └── release.yml # Optional, depends on legal requirements +├── raw/ +├── staging/ +├── resources/ +├── releases/ +├── src/ +│ └── / +│ ├── __init__.py +│ └── build.py +├── . +├── CHANGELOG.md +├── README.md +├── LICENSE.md +└── +``` + +::: callout-tip +Why do we "build" a data package? We build it for several reasons. + +1. To keep a separation between the source code and raw input data contained + within the Git repository and the final data package that is released. +2. To ensure a reproducible build from input to output. +3. To keep a history of the changes made to the final data and to version the + data package for easier tracking of changes and updates. +4. To treat the data package as a formal "product" and apply rigorous and robust + software engineering and data engineering practices to its development and + release. +::: + +## Steps + +The initial steps, from the source of the data to staging, is managed by a +pipeline management tool, which should have its pipeline defined in +`src//build.py` (though some pipeline tools require a different +file name). We recommend using +[pytask](https://pytask-dev.readthedocs.io/en/stable/), which we've found works +well for developing data packages. All steps must be defined within this tool, +from pulling the source to raw, to processing the raw data into staging, and to +processing or extracting any metadata obtained from the source or staged data. + +The next steps are managed mainly by +[Sprout](https://sprout.seedcase-project.org/), which is a tool that handles the +processing from staging to resources and for (re-)generating the metadata file. + +The final steps for building the data package into `.tar` and `.zip` files are +handled through the [`justfile`](https://just.systems/) file, which would be +`` in the above directory structure. The `justfile` contains +recipes for each step of the build process and those steps are executed in order +so that the data package is built. + +If the data package is based on open data or on data not covered by legal +restrictions (e.g. GDPR), the build and release process can be automated through +the GitHub Actions workflow defined in `.github/workflows/release.yml`. + +```{mermaid} +%%| label: fig-build-process +%%| fig-cap: "The process for building a data package." +%%| fig-alt: A flowchart showing the process for building a data package. The +%%| process starts with the data from the source being pulled into the `raw/` +%%| directory. The data in `raw/` is then processed into `staging/`. The data +%%| in `staging/` is then joined into `resources/`. The metadata file is +%%| optionally regenerated from the code and checked against the data in +%%| `staging/`. Finally, the metadata and resources are bundled into a `.tar` +%%| and `.zip` file in `releases/`. + +flowchart TB + Source["source
(e.g., API,
database)"] -->|pull| Raw["raw/"] + Raw -->|process| Staging["staging/"] + Staging -->|join| Resources["resources/"] + Resources + Metadata["metadata
(Optionally
regenerated)"] + Metadata ---|check| Staging + Metadata & Resources -->|bundled| Build["releases/
.tar and .zip"] +``` + +::: callout-note +We plan on building a tool to automate and simplify the build step, but for now, +use the structure and tools that we've set up in our [Template Data +Package](https://template-data-package.seedcase-project.org/). See @sec-setup +for details about that. +::: + +::: callout-important +It's important to note here that only the data in `raw/` and the metadata file +are saved into [Git LFS](https://git-lfs.github.com/) during the release +process. No other data artifacts or files are saved in the Git history. This is +described in more detail in @sec-release. +::: + +### Pull from source to raw + +To make sure we're getting the most up-to-date data, the first build step is to +pull from the various data sources into `raw/`. See @sec-raw and @sec-src for +details on how the raw data is organised and how the source code pulls the data +via the pipeline tool. The data is *not* processed in any way when it is pulled +from the source. + +### Raw to staging + +After pulling the source data, the data (usually) needs to be processed in some +way. In particular, the data likely needs to be reorganised so that it ends up +in the correct resource location. We process the data in `raw/` into `staging/` +in a one-to-one mapping, so that every `.` file in `raw/` +is processed into a `.parquet` file in `staging/`. This enables the +potential for parallel processing and to maintain a sequence between the two +directories. Any data in `staging/` must also match the contents of the metadata +format. This is all done to ensure that the data gets into the final resource +correctly and without issues. See @sec-staging, @sec-metadata, and @sec-src for +more details `staging/`. + +### Staging to resources + +At this stage, Sprout takes over and processes the data from `staging/` into +`resources/`. All the timestamped files in the individual `staging/` resource +directories are processed and joined into a single resource file in +`resources/`. During processing, the data is checked against the metadata. See +@sec-resources, @sec-metadata, and [Sprout's +documentation](https://sprout.seedcase-project.org/) for more details on this +step. + +### Metadata (re-)generation (optional) + +For data packages using `datapackage.json` as the metadata format, Sprout will +also (re-)generate the file from the Python code. See @sec-metadata and +[Sprout's documentation](https://sprout.seedcase-project.org/) for a description +of this section. + +### Build artifacts + +At the end, the data package is built into one `_.tar` +file that contains the metadata file (e.g. `datapackage.json`), `LICENSE.md`, +`README.md`, `CHANGELOG.md`, and the resource files. If the data contains human +(especially health) data and if it is required to be on secure servers, an +additional `_.zip` file is also built with the same files +except for the data. This `.zip` file can be uploaded to public archives to +generate, for example, a [DOI](https://doi.org/) on +[Zenodo](https://zenodo.org), while the `.tar` file can remain on the server. +The `.tar` and `.zip` files are saved into a Git ignored `releases/` directory. + + + +## Development practices + +How does this build process impact developing the data package and the +development practices? Because of this formal build process, there are a few +things to do differently. + +First, you shouldn't manually store any data in the Git history within the Git +LFS whenever you make changes and submit a pull request. Treat any data pulled +from sources or processed into staging or resources as temporary. Only the build +process should store any data or metadata in the Git history. + +Related to above, if the metadata format is `datapackage.json`, pull requests +should *not* contain any changes to it, nor should they contain any changes or +additions of data in `raw/`, `staging/`, or `resources/`. These files are +generated during the build process and should not be modified or added directly. +For `datapackage.json` based metadata, the metadata files within `src/` that +contain the metadata managed by [Sprout](https://sprout.seedcase-project.org/) +should be modified instead. For other metadata formats, the metadata file can be +modified directly and can be saved to the Git history. + +Commit messages should still be written in the Conventional Commits format, +though the specific commit types used are a bit different considering no data, +or metadata files if it is the `datapackage.json` format, are being modified or +saved directly. See @sec-release for more details on what commit messages to +use. + +All files in `staging/`, `resources/`, and `releases/` should be ignored by Git +in the `.gitignore` file, aside from a `.gitkeep` or `README.md` file to keep +the directory structure. From 6cc482ebb9764a295912aa3327a7bebb18920163 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:09:05 +0000 Subject: [PATCH 07/11] =?UTF-8?q?chore:=20=E2=9C=8F=EF=B8=8F=20automatic?= =?UTF-8?q?=20pre-commit=20hook=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.qmd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/index.qmd b/index.qmd index 49bb6b8..45d88a4 100644 --- a/index.qmd +++ b/index.qmd @@ -28,9 +28,9 @@ the software package development cycle so that the final "data package" (or This guide mostly follows the [diátaxis](https://diataxis.fr/how-to-guides/) "how-to guide" style, though not strictly. It is a living and constantly evolving guide that is regularly updated as we learn and refine how we work and -develop data packages. We intend to continually update and release it with -every update to Zenodo and as GitHub releases. We don't expect this guide to -ever be considered "done". +develop data packages. We intend to continually update and release it with every +update to Zenodo and as GitHub releases. We don't expect this guide to ever be +considered "done". ::: ## Who you are @@ -67,8 +67,8 @@ the GitHub repository. ::: content-hidden -The PDF version of the guide is available in the releases page, as well as -the Zenodo archive. +The PDF version of the guide is available in the releases page, as well as the +Zenodo archive. ::: ## How the website is made From 1b958255be81b92186c4409b39e0eb730f15f371 Mon Sep 17 00:00:00 2001 From: "Luke W. Johnston" Date: Tue, 14 Jul 2026 15:14:55 +0200 Subject: [PATCH 08/11] refactor: :pencil2: edits from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Signe Kirk Brødbæk <40836345+signekb@users.noreply.github.com> --- docs/build.qmd | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/build.qmd b/docs/build.qmd index 42ac2e7..d187ba1 100644 --- a/docs/build.qmd +++ b/docs/build.qmd @@ -1,11 +1,11 @@ # Build process {#sec-build} -Like other types of packages (e.g. Rust, Python, R), the contents of the Git -repository *build* the final data package, but aren't the data package itself. -The repository contains the source of the package, such as the raw data input -and the code to process the data into the final format. But it isn't the package -itself. We need to first "compile" the package by running the code to take the -raw data input and build the final data package. +Like software packages (e.g. in Rust, Python, R), the Git +repository +contains the source of the package rather than the package itself. It includes the raw input data +and the code needed to process it into the final format, but before the data package can be distributed, it +must first be "built" (or "compiled") by running the code to transform the +raw data inputs to the final packaged format. At a high level, the directories and files involved in the build process are (non-relevant files and directories are removed): @@ -30,8 +30,8 @@ At a high level, the directories and files involved in the build process are └── ``` -::: callout-tip -Why do we "build" a data package? We build it for several reasons. +::: callout-note +Why do we "build" a data package? 1. To keep a separation between the source code and raw input data contained within the Git repository and the final data package that is released. @@ -50,23 +50,22 @@ pipeline management tool, which should have its pipeline defined in `src//build.py` (though some pipeline tools require a different file name). We recommend using [pytask](https://pytask-dev.readthedocs.io/en/stable/), which we've found works -well for developing data packages. All steps must be defined within this tool, +well for developing data packages (for more details, see our [pytask for Python workflow management](https://decisions.seedcase-project.org/why-pytask/) decision post). All steps must be defined within this tool, from pulling the source to raw, to processing the raw data into staging, and to processing or extracting any metadata obtained from the source or staged data. The next steps are managed mainly by -[Sprout](https://sprout.seedcase-project.org/), which is a tool that handles the +[Sprout](https://sprout.seedcase-project.org/), which is our tool to handle the processing from staging to resources and for (re-)generating the metadata file. The final steps for building the data package into `.tar` and `.zip` files are handled through the [`justfile`](https://just.systems/) file, which would be -`` in the above directory structure. The `justfile` contains -recipes for each step of the build process and those steps are executed in order -so that the data package is built. +`` in the above directory structure. The `justfile`, if you use our [template data package](https://template-data-package.seedcase-project.org/), contains +recipes for each step of the build process. If the data package is based on open data or on data not covered by legal restrictions (e.g. GDPR), the build and release process can be automated through -the GitHub Actions workflow defined in `.github/workflows/release.yml`. +a GitHub Actions workflow defined in `.github/workflows/release.yml`. ```{mermaid} %%| label: fig-build-process @@ -80,18 +79,18 @@ the GitHub Actions workflow defined in `.github/workflows/release.yml`. %%| and `.zip` file in `releases/`. flowchart TB - Source["source
(e.g., API,
database)"] -->|pull| Raw["raw/"] + Source["source
(e.g., API
or database)"] -->|pull| Raw["raw/"] Raw -->|process| Staging["staging/"] Staging -->|join| Resources["resources/"] Resources Metadata["metadata
(Optionally
regenerated)"] Metadata ---|check| Staging - Metadata & Resources -->|bundled| Build["releases/
.tar and .zip"] + Metadata & Resources -->|bundle| Build["releases/
.tar and .zip"] ``` ::: callout-note -We plan on building a tool to automate and simplify the build step, but for now, -use the structure and tools that we've set up in our [Template Data +We plan on building a tool to automate and simplify the build steps, but for now, +you can use the structure and tools that we've set up in our [Template Data Package](https://template-data-package.seedcase-project.org/). See @sec-setup for details about that. ::: @@ -109,13 +108,13 @@ To make sure we're getting the most up-to-date data, the first build step is to pull from the various data sources into `raw/`. See @sec-raw and @sec-src for details on how the raw data is organised and how the source code pulls the data via the pipeline tool. The data is *not* processed in any way when it is pulled -from the source. +from the source into `raw/`. ### Raw to staging -After pulling the source data, the data (usually) needs to be processed in some +After pulling the source data into the `raw/` directory, the data (usually) needs to be processed in some way. In particular, the data likely needs to be reorganised so that it ends up -in the correct resource location. We process the data in `raw/` into `staging/` +in the correct resource location. The data in `raw/` should be processed into `staging/` in a one-to-one mapping, so that every `.` file in `raw/` is processed into a `.parquet` file in `staging/`. This enables the potential for parallel processing and to maintain a sequence between the two @@ -126,7 +125,7 @@ more details `staging/`. ### Staging to resources -At this stage, Sprout takes over and processes the data from `staging/` into +At this stage, [Sprout](https://sprout.seedcase-project.org/) can take over and process the data from `staging/` into `resources/`. All the timestamped files in the individual `staging/` resource directories are processed and joined into a single resource file in `resources/`. During processing, the data is checked against the metadata. See @@ -151,15 +150,15 @@ additional `_.zip` file is also built with the same files except for the data. This `.zip` file can be uploaded to public archives to generate, for example, a [DOI](https://doi.org/) on [Zenodo](https://zenodo.org), while the `.tar` file can remain on the server. -The `.tar` and `.zip` files are saved into a Git ignored `releases/` directory. +The `.tar` and `.zip` files are saved into a `releases/` directory ignored by Git. - + ## Development practices How does this build process impact developing the data package and the development practices? Because of this formal build process, there are a few -things to do differently. +things to be aware of. First, you shouldn't manually store any data in the Git history within the Git LFS whenever you make changes and submit a pull request. Treat any data pulled From 29a284494c305edce5338acda2a01489b26e30f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:15:01 +0000 Subject: [PATCH 09/11] =?UTF-8?q?chore:=20=E2=9C=8F=EF=B8=8F=20automatic?= =?UTF-8?q?=20pre-commit=20hook=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/build.qmd | 64 +++++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/docs/build.qmd b/docs/build.qmd index d187ba1..4c43217 100644 --- a/docs/build.qmd +++ b/docs/build.qmd @@ -1,11 +1,10 @@ # Build process {#sec-build} -Like software packages (e.g. in Rust, Python, R), the Git -repository -contains the source of the package rather than the package itself. It includes the raw input data -and the code needed to process it into the final format, but before the data package can be distributed, it -must first be "built" (or "compiled") by running the code to transform the -raw data inputs to the final packaged format. +Like software packages (e.g. in Rust, Python, R), the Git repository contains +the source of the package rather than the package itself. It includes the raw +input data and the code needed to process it into the final format, but before +the data package can be distributed, it must first be "built" (or "compiled") by +running the code to transform the raw data inputs to the final packaged format. At a high level, the directories and files involved in the build process are (non-relevant files and directories are removed): @@ -31,7 +30,7 @@ At a high level, the directories and files involved in the build process are ``` ::: callout-note -Why do we "build" a data package? +Why do we "build" a data package? 1. To keep a separation between the source code and raw input data contained within the Git repository and the final data package that is released. @@ -50,9 +49,11 @@ pipeline management tool, which should have its pipeline defined in `src//build.py` (though some pipeline tools require a different file name). We recommend using [pytask](https://pytask-dev.readthedocs.io/en/stable/), which we've found works -well for developing data packages (for more details, see our [pytask for Python workflow management](https://decisions.seedcase-project.org/why-pytask/) decision post). All steps must be defined within this tool, -from pulling the source to raw, to processing the raw data into staging, and to -processing or extracting any metadata obtained from the source or staged data. +well for developing data packages (for more details, see our [pytask for Python +workflow management](https://decisions.seedcase-project.org/why-pytask/) +decision post). All steps must be defined within this tool, from pulling the +source to raw, to processing the raw data into staging, and to processing or +extracting any metadata obtained from the source or staged data. The next steps are managed mainly by [Sprout](https://sprout.seedcase-project.org/), which is our tool to handle the @@ -60,8 +61,9 @@ processing from staging to resources and for (re-)generating the metadata file. The final steps for building the data package into `.tar` and `.zip` files are handled through the [`justfile`](https://just.systems/) file, which would be -`` in the above directory structure. The `justfile`, if you use our [template data package](https://template-data-package.seedcase-project.org/), contains -recipes for each step of the build process. +`` in the above directory structure. The `justfile`, if you use our +[template data package](https://template-data-package.seedcase-project.org/), +contains recipes for each step of the build process. If the data package is based on open data or on data not covered by legal restrictions (e.g. GDPR), the build and release process can be automated through @@ -89,8 +91,8 @@ flowchart TB ``` ::: callout-note -We plan on building a tool to automate and simplify the build steps, but for now, -you can use the structure and tools that we've set up in our [Template Data +We plan on building a tool to automate and simplify the build steps, but for +now, you can use the structure and tools that we've set up in our [Template Data Package](https://template-data-package.seedcase-project.org/). See @sec-setup for details about that. ::: @@ -112,24 +114,25 @@ from the source into `raw/`. ### Raw to staging -After pulling the source data into the `raw/` directory, the data (usually) needs to be processed in some -way. In particular, the data likely needs to be reorganised so that it ends up -in the correct resource location. The data in `raw/` should be processed into `staging/` -in a one-to-one mapping, so that every `.` file in `raw/` -is processed into a `.parquet` file in `staging/`. This enables the -potential for parallel processing and to maintain a sequence between the two -directories. Any data in `staging/` must also match the contents of the metadata -format. This is all done to ensure that the data gets into the final resource -correctly and without issues. See @sec-staging, @sec-metadata, and @sec-src for -more details `staging/`. +After pulling the source data into the `raw/` directory, the data (usually) +needs to be processed in some way. In particular, the data likely needs to be +reorganised so that it ends up in the correct resource location. The data in +`raw/` should be processed into `staging/` in a one-to-one mapping, so that +every `.` file in `raw/` is processed into a +`.parquet` file in `staging/`. This enables the potential for +parallel processing and to maintain a sequence between the two directories. Any +data in `staging/` must also match the contents of the metadata format. This is +all done to ensure that the data gets into the final resource correctly and +without issues. See @sec-staging, @sec-metadata, and @sec-src for more details +`staging/`. ### Staging to resources -At this stage, [Sprout](https://sprout.seedcase-project.org/) can take over and process the data from `staging/` into -`resources/`. All the timestamped files in the individual `staging/` resource -directories are processed and joined into a single resource file in -`resources/`. During processing, the data is checked against the metadata. See -@sec-resources, @sec-metadata, and [Sprout's +At this stage, [Sprout](https://sprout.seedcase-project.org/) can take over and +process the data from `staging/` into `resources/`. All the timestamped files in +the individual `staging/` resource directories are processed and joined into a +single resource file in `resources/`. During processing, the data is checked +against the metadata. See @sec-resources, @sec-metadata, and [Sprout's documentation](https://sprout.seedcase-project.org/) for more details on this step. @@ -150,7 +153,8 @@ additional `_.zip` file is also built with the same files except for the data. This `.zip` file can be uploaded to public archives to generate, for example, a [DOI](https://doi.org/) on [Zenodo](https://zenodo.org), while the `.tar` file can remain on the server. -The `.tar` and `.zip` files are saved into a `releases/` directory ignored by Git. +The `.tar` and `.zip` files are saved into a `releases/` directory ignored by +Git. From 24f9e537a8d638779781d448ff63b6ce90a613a0 Mon Sep 17 00:00:00 2001 From: "Luke W. Johnston" Date: Tue, 14 Jul 2026 15:15:25 +0200 Subject: [PATCH 10/11] refactor: :pencil2: small edit to clarify sentence --- docs/build.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build.qmd b/docs/build.qmd index 4c43217..0898bb1 100644 --- a/docs/build.qmd +++ b/docs/build.qmd @@ -181,7 +181,7 @@ modified directly and can be saved to the Git history. Commit messages should still be written in the Conventional Commits format, though the specific commit types used are a bit different considering no data, or metadata files if it is the `datapackage.json` format, are being modified or -saved directly. See @sec-release for more details on what commit messages to +saved directly. See @sec-release for more details on how Conventional Commits are used in the release process and what commit messages to use. All files in `staging/`, `resources/`, and `releases/` should be ignored by Git From ed33077672391f454e45ad3119be0acfcf67f53f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:15:32 +0000 Subject: [PATCH 11/11] =?UTF-8?q?chore:=20=E2=9C=8F=EF=B8=8F=20automatic?= =?UTF-8?q?=20pre-commit=20hook=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/build.qmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build.qmd b/docs/build.qmd index 0898bb1..7c67512 100644 --- a/docs/build.qmd +++ b/docs/build.qmd @@ -181,8 +181,8 @@ modified directly and can be saved to the Git history. Commit messages should still be written in the Conventional Commits format, though the specific commit types used are a bit different considering no data, or metadata files if it is the `datapackage.json` format, are being modified or -saved directly. See @sec-release for more details on how Conventional Commits are used in the release process and what commit messages to -use. +saved directly. See @sec-release for more details on how Conventional Commits +are used in the release process and what commit messages to use. All files in `staging/`, `resources/`, and `releases/` should be ignored by Git in the `.gitignore` file, aside from a `.gitkeep` or `README.md` file to keep