Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v4
uses: actions/checkout@v6

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
Expand All @@ -62,7 +62,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
uses: github/codeql-action/autobuild@v4

# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
Expand All @@ -75,6 +75,6 @@ jobs:
# ./location_of_script_within_repo/buildscript.sh

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v4
with:
category: "/language:${{matrix.language}}"
5 changes: 4 additions & 1 deletion .github/workflows/django-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@ on:
jobs:
run_tests:
runs-on: ubuntu-latest
permissions:
contents: read
env:
DEPLOY_ENVIRONMENT: test
DOCKER_BUILD_FLAGS: --quiet
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- name: build containers and run Django tests
run: make test
8 changes: 5 additions & 3 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,18 @@ on:
jobs:
e2e:
runs-on: ubuntu-latest
permissions:
contents: read
env:
DEPLOY_ENVIRONMENT: test
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6

- name: deploy services in e2e mode
run: make e2e

- name: run e2e tests
uses: cypress-io/github-action@v6
uses: cypress-io/github-action@v7
with:
working-directory: e2e
wait-on: "http://localhost:8000"
Expand All @@ -30,7 +32,7 @@ jobs:

- name: upload cypress videos
if: failure()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v7
with:
name: cypress-videos
path: e2e/cypress/videos
33 changes: 20 additions & 13 deletions .github/workflows/frontend-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,44 @@ name: Frontend CI (Vue3/TS)
on:
push:
branches: [main]
paths: ["frontend/**"]
paths: ["frontend/**", ".github/workflows/frontend-ci.yml"]
pull_request:
branches: [main]
paths: ["frontend/**"]
paths: ["frontend/**", ".github/workflows/frontend-ci.yml"]

concurrency:
group: frontend-ci-${{ github.ref }}
cancel-in-progress: true

jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
env:
DEPLOY_ENVIRONMENT: dev
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- name: build container
run: |
make docker-compose.yml
docker compose build vite --pull --no-cache
docker compose build vite --pull --no-cache --quiet
docker compose up vite -d
- name: Lint
run: docker compose exec vite npm run lint
run: docker compose exec -T vite npm run lint
- name: Check formatting
if: ${{ success() || failure() }}
run: docker compose exec vite npm run style
if: ${{ always() }}
run: docker compose exec -T vite npm run style
- name: Unit tests
if: ${{ success() || failure() }}
run: docker compose exec vite npm run test
if: ${{ always() }}
run: docker compose exec -T vite npm run test
- name: Type check
if: ${{ success() || failure() }}
run: docker compose exec vite npm run type-check
if: ${{ always() }}
run: docker compose exec -T vite npm run type-check
- name: Build for production
if: ${{ success() || failure() }}
run: docker compose exec vite npm run build
if: ${{ always() }}
run: docker compose exec -T vite npm run build
- name: bring down container
if: ${{ always() }}
run: docker compose down
2 changes: 2 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ This platform manages scientific software artifacts and publication metadata. Th
- Add or update tests for behavior changes, bug fixes, and nontrivial logic
- Prefer targeted test execution during development
- Use repository-standard containerized commands for running tests and tooling
- For targeted Django tests, run exactly: `make test TEST_ARGS=<dotted.test.path>`
- If `make test` fails in WSL with Docker daemon or credential-helper errors, prompt the user to start Docker Desktop for Windows and confirm before retrying
- Do not change existing behavior without updating or adding tests

## Environment and Commands
Expand Down
37 changes: 19 additions & 18 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
ENVREPLACE := deploy/scripts/envreplace
DEPLOY_CONF_DIR=deploy/conf
ENV_TEMPLATE=${DEPLOY_CONF_DIR}/.env.template
DOCKER_BUILD_FLAGS ?=

# assumes a .tar.xz file
BORG_REPO_URL := https://example.com/repo.tar.xz
Expand All @@ -32,29 +33,29 @@ include .env

.PHONY: build
build: docker-compose.yml secrets $(DOCKER_SHARED_DIR)
docker compose --progress=plain build --pull --parallel
@docker compose build --pull --parallel $(DOCKER_BUILD_FLAGS)

$(BORG_REPO_PATH):
wget -c ${BORG_REPO_URL} -P ${BUILD_DIR}

config.mk:
envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk
@envsubst < ${DEPLOY_CONF_DIR}/config.mk.template > config.mk

.PHONY: $(DOCKER_SHARED_DIR)
$(DOCKER_SHARED_DIR):
for d in ${DOCKER_SHARED_SUBDIRS} ; do \
@for d in ${DOCKER_SHARED_SUBDIRS} ; do \
mkdir -p ${DOCKER_SHARED_DIR}/$$d ; \
done

${SECRETS_DIR}:
mkdir -p ${SECRETS_DIR}
@mkdir -p ${SECRETS_DIR}

$(SECRET_KEY_PATH): | ${SECRETS_DIR}
SECRET_KEY=$$(openssl rand -base64 48); \
@SECRET_KEY=$$(openssl rand -base64 48); \
echo "$${SECRET_KEY}" > $(SECRET_KEY_PATH)

$(DB_PASSWORD_PATH): | ${SECRETS_DIR}
DB_PASSWORD=$$(openssl rand -base64 48); \
@DB_PASSWORD=$$(openssl rand -base64 48); \
TODAY=$$(date +%Y-%m-%d-%H:%M:%S); \
if [ -f $(DB_PASSWORD_PATH) ]; \
then \
Expand All @@ -64,15 +65,15 @@ $(DB_PASSWORD_PATH): | ${SECRETS_DIR}
@echo "db password at $(DB_PASSWORD_PATH) was reset, may need to manually update existing db password"

$(PGPASS_PATH): $(DB_PASSWORD_PATH) | ${SECRETS_DIR}
echo "${DB_HOST}:5432:*:${DB_USER}:$$(cat $(DB_PASSWORD_PATH))" > $(PGPASS_PATH)
chmod 0600 $(PGPASS_PATH)
@echo "${DB_HOST}:5432:*:${DB_USER}:$$(cat $(DB_PASSWORD_PATH))" > $(PGPASS_PATH)
@chmod 0600 $(PGPASS_PATH)

.PHONY: release-version
release-version: .env
$(ENVREPLACE) RELEASE_VERSION $$(git describe --tags --abbrev=1) .env
@$(ENVREPLACE) RELEASE_VERSION $$(git describe --tags --abbrev=1 2>/dev/null || git rev-parse --short HEAD) .env

.env: $(DB_PASSWORD_PATH) $(SECRET_KEY_PATH)
if [ ! -f .env ]; then \
@if [ ! -f .env ]; then \
cp $(ENV_TEMPLATE) .env; \
fi; \
# $(ENVREPLACE) DB_PASSWORD $$(cat $(DB_PASSWORD_PATH)) .env; \
Expand All @@ -81,7 +82,7 @@ release-version: .env

.PHONY: docker-compose.yml
docker-compose.yml: base.yml dev.yml staging.yml test.yml prod.yml config.mk $(PGPASS_PATH) release-version .env
case "$(DEPLOY_ENVIRONMENT)" in \
@case "$(DEPLOY_ENVIRONMENT)" in \
dev|staging|test) docker compose -f base.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
prod) docker compose -f base.yml -f staging.yml -f $(DEPLOY_ENVIRONMENT).yml config > docker-compose.yml;; \
*) echo "invalid environment. must be either dev, staging or prod" 1>&2; exit 1;; \
Expand All @@ -93,17 +94,17 @@ set-db-password: $(DB_PASSWORD_PATH) .env

.PHONY: secrets
secrets: $(SECRETS_DIR) $(GENERATED_SECRETS)
for secret_path in $(EXT_SECRETS); do \
@for secret_path in $(EXT_SECRETS); do \
touch ${SECRETS_DIR}/$$secret_path; \
done

.PHONY: deploy
deploy: build
docker compose pull db redis elasticsearch
docker compose pull -q db redis elasticsearch
ifneq ($(DEPLOY_ENVIRONMENT),dev)
docker compose pull nginx
docker compose pull -q nginx
endif
docker compose up -d
docker compose up -d --quiet-pull
sleep 42
docker compose exec server inv prepare

Expand All @@ -122,7 +123,7 @@ restore: build $(BORG_REPO_PATH) | $(REPO_BACKUPS_PATH)
@echo "Backing existing ${REPO_BACKUPS_PATH}/repo to fresh mktemp directory in /tmp"
sudo mv $(REPO_BACKUPS_PATH)/repo $(shell mktemp -d)
tar -Jxf $(BORG_REPO_PATH) -C $(REPO_BACKUPS_PATH)
docker compose up -d
docker compose up -d --quiet-pull
docker compose exec server inv borg.restore

.PHONY: clean
Expand All @@ -136,7 +137,7 @@ clean_deploy: clean

.PHONY: test
test: build
docker compose run --rm server /code/deploy/test.sh $(TEST_ARGS)
docker compose run --quiet-pull --rm server /code/deploy/test.sh $(TEST_ARGS)

# e2e testing setup

Expand All @@ -152,7 +153,7 @@ $(E2E_REPO_PATH):
.PHONY: e2e
e2e: docker-compose.yml secrets $(DOCKER_SHARED_DIR) $(E2E_REPO_PATH)
docker compose -f docker-compose.yml -f e2e.yml build -q
docker compose -f docker-compose.yml -f e2e.yml up -d
docker compose -f docker-compose.yml -f e2e.yml up -d --quiet-pull
sleep 42
docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\
inv borg.restore --force && \
Expand Down
2 changes: 1 addition & 1 deletion base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ services:
db:
condition: service_healthy
elasticsearch:
condition: service_started
condition: service_healthy
redis:
condition: service_started
vite:
Expand Down
12 changes: 6 additions & 6 deletions django/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ RUN --mount=type=cache,target=/var/lib/apt,sharing=locked \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
DEBIAN_FRONTEND=noninteractive \
sed -i "s|archive.ubuntu.com|${UBUNTU_MIRROR}|" /etc/apt/sources.list \
&& apt-get -q update \
&& apt-get -q install -y postgresql-common --no-install-recommends \
&& apt-get -qq update \
&& apt-get -qq install -y --no-install-recommends -o=Dpkg::Use-Pty=0 postgresql-common \
&& yes | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh \
&& apt-get -q update \
&& apt-get -q install -y --no-install-recommends \
&& apt-get -qq update \
&& apt-get -qq install -y --no-install-recommends -o=Dpkg::Use-Pty=0 \
autopostgresqlbackup \
binutils \
borgbackup \
Expand Down Expand Up @@ -44,7 +44,7 @@ RUN --mount=type=cache,target=/var/lib/apt,sharing=locked \
unzip \
&& update-alternatives --install /usr/bin/python python /usr/bin/python3 1000 \
&& python -m venv ${VIRTUAL_ENV} \
&& apt-get upgrade -q -y -o Dpkg::Options::="--force-confold" \
&& apt-get -qq upgrade -y -o Dpkg::Use-Pty=0 -o Dpkg::Options::="--force-confold" \
&& mkdir -p /etc/service/django /etc/service/huey \
&& touch /etc/service/django/run /etc/service/huey/run /etc/postgresql-backup-pre \
&& chmod a+x /etc/service/django/run /etc/service/huey/run /etc/postgresql-backup-pre \
Expand All @@ -55,7 +55,7 @@ COPY ${REQUIREMENTS_FILE} requirements.txt /tmp/

# FIXME: change to container user
RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked \
pip install -r /tmp/${REQUIREMENTS_FILE}
pip install --progress-bar off -r /tmp/${REQUIREMENTS_FILE}

COPY ./deploy/cron.daily/* /etc/cron.daily/
COPY ./deploy/cron.hourly/* /etc/cron.hourly/
Expand Down
8 changes: 8 additions & 0 deletions django/core/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,14 @@ class Industry(models.TextChoices):
index.FilterField("name"),
],
),
index.RelatedFields(
"user",
[
index.FilterField("date_joined"),
index.FilterField("is_active"),
index.FilterField("username"),
],
),
]

# Proxies to related user object
Expand Down
23 changes: 18 additions & 5 deletions django/core/settings/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,29 @@
DEPLOY_ENVIRONMENT, WAGTAILADMIN_BASE_URL, BASE_URL = set_environment(Environment.TEST)
TESTING = True

ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server"]
ALLOWED_HOSTS = ["localhost", "127.0.0.1", "server", "testserver"]

EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

LOGGING["loggers"]["core.views"] = {
"level": "ERROR",
"handlers": ["console"],
"propagate": False,
logger_levels = {
# Keep dependency logs focused on actionable warnings/errors in tests.
"django_tasks": "WARNING",
"urllib3": "WARNING",
"elastic_transport": "WARNING",
"invoke": "WARNING",
# Keep application logs concise without hiding warnings that may indicate regressions.
"core": "WARNING",
"library": "WARNING",
"home": "WARNING",
}

for logger_name, logger_level in logger_levels.items():
LOGGING["loggers"][logger_name] = {
"level": logger_level,
"handlers": ["console"],
"propagate": False,
}

SHARE_DIR = path.realpath("/shared/tests")
LIBRARY_ROOT = path.join(SHARE_DIR, "library")
LIBRARY_PREVIOUS_ROOT = path.join(SHARE_DIR, ".latest")
Expand Down
5 changes: 4 additions & 1 deletion django/core/view_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,10 @@ def get_search_queryset(
search_backend = get_search_backend()

if not fields:
fields = []
# Passing an explicit empty list can be interpreted by some backends as
# "search across no fields", yielding empty results. Use None to allow
# backend defaults.
fields = None

if not tags:
tags = []
Expand Down
3 changes: 2 additions & 1 deletion django/curator/invoke_tasks/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ def deny_robots(ctx):

@task(aliases=["cs"])
def collectstatic(ctx, reload_uwsgi=False):
dj(ctx, "collectstatic -c --noinput", pty=True)
# Keep collectstatic output concise; errors still surface with verbosity 0.
dj(ctx, "collectstatic -c --noinput --verbosity 0", pty=True)
if reload_uwsgi:
restart_uwsgi(ctx)

Expand Down
Loading
Loading