From 9cd872013bfc30dbe1bbd1c098035a0463180586 Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Fri, 13 Feb 2026 10:51:36 -0500 Subject: [PATCH] Add measures for catching flaky tests --- .github/workflows/ci.yml | 72 +-------------------- .github/workflows/nightly-flaky-tests.yml | 12 ++++ .github/workflows/test.yml | 77 +++++++++++++++++++++++ isic/core/tests/test_isic_oauth_app.py | 1 + isic/core/tests/test_storage.py | 7 ++- 5 files changed, 98 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/nightly-flaky-tests.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 452eabeae..b7644f74d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,75 +12,9 @@ concurrency: cancel-in-progress: true jobs: test: - runs-on: ubuntu-latest - services: - postgres: - image: pgvector/pgvector:pg17 - env: - POSTGRES_DB: django - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd "pg_isready --username postgres" - --health-start-period 30s - --health-start-interval 2s - ports: - - 5432:5432 - elasticsearch: - image: elasticsearch:7.17.28 - env: - ES_JAVA_OPTS: "-Xms250m -Xmx750m" - discovery.type: single-node - xpack.security.enabled: "true" - ELASTIC_PASSWORD: elastic - options: >- - --health-cmd "curl --fail --user elastic:elastic http://localhost:9200/" - --health-start-period 30s - --health-start-interval 2s - ports: - - 9200:9200 - rabbitmq: - image: rabbitmq:management-alpine - options: >- - --health-cmd "rabbitmq-diagnostics ping" - --health-start-period 30s - --health-start-interval 2s - ports: - - 5672:5672 - minio: - # This image does not require any command arguments (which GitHub Actions don't support) - image: bitnamilegacy/minio:latest - env: - MINIO_ROOT_USER: minioAccessKey - MINIO_ROOT_PASSWORD: minioSecretKey - options: >- - --health-cmd "mc ready local" - --health-timeout 1s - --health-start-period 30s - --health-start-interval 2s - ports: - - 9000:9000 - redis: - image: redis:alpine - options: >- - --health-cmd "redis-cli ping" - --health-start-period 30s - --health-start-interval 2s - ports: - - 6379:6379 - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - name: Install uv - uses: astral-sh/setup-uv@v7 - - name: Run tests - run: | - uv run tox - env: - DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django - DJANGO_ISIC_ELASTICSEARCH_URL: http://elastic:elastic@localhost:9200 - DJANGO_CELERY_BROKER_URL: amqp://localhost:5672/ - DJANGO_MINIO_STORAGE_URL: http://minioAccessKey:minioSecretKey@localhost:9000/django-storage - DJANGO_CACHE_URL: redis://localhost:6379/0 + uses: ./.github/workflows/test.yml + with: + tox-command: uv run tox build-node: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/nightly-flaky-tests.yml b/.github/workflows/nightly-flaky-tests.yml new file mode 100644 index 000000000..8428ad8d2 --- /dev/null +++ b/.github/workflows/nightly-flaky-tests.yml @@ -0,0 +1,12 @@ +name: nightly-flaky-tests +on: + schedule: + - cron: "0 6 * * *" + workflow_dispatch: +permissions: + contents: read +jobs: + test: + uses: ./.github/workflows/test.yml + with: + tox-command: uv run tox -e test -- --count 5 --random-order diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..92eeb63e6 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,77 @@ +name: test +on: + workflow_call: + inputs: + tox-command: + required: true + type: string +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_DB: django + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd "pg_isready --username postgres" + --health-start-period 30s + --health-start-interval 2s + ports: + - 5432:5432 + elasticsearch: + image: elasticsearch:7.17.28 + env: + ES_JAVA_OPTS: "-Xms250m -Xmx750m" + discovery.type: single-node + xpack.security.enabled: "true" + ELASTIC_PASSWORD: elastic + options: >- + --health-cmd "curl --fail --user elastic:elastic http://localhost:9200/" + --health-start-period 30s + --health-start-interval 2s + ports: + - 9200:9200 + rabbitmq: + image: rabbitmq:management-alpine + options: >- + --health-cmd "rabbitmq-diagnostics ping" + --health-start-period 30s + --health-start-interval 2s + ports: + - 5672:5672 + minio: + # This image does not require any command arguments (which GitHub Actions don't support) + image: bitnamilegacy/minio:latest + env: + MINIO_ROOT_USER: minioAccessKey + MINIO_ROOT_PASSWORD: minioSecretKey + options: >- + --health-cmd "mc ready local" + --health-timeout 1s + --health-start-period 30s + --health-start-interval 2s + ports: + - 9000:9000 + redis: + image: redis:alpine + options: >- + --health-cmd "redis-cli ping" + --health-start-period 30s + --health-start-interval 2s + ports: + - 6379:6379 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Run tests + run: ${{ inputs.tox-command }} + env: + DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django + DJANGO_ISIC_ELASTICSEARCH_URL: http://elastic:elastic@localhost:9200 + DJANGO_CELERY_BROKER_URL: amqp://localhost:5672/ + DJANGO_MINIO_STORAGE_URL: http://minioAccessKey:minioSecretKey@localhost:9000/django-storage + DJANGO_CACHE_URL: redis://localhost:6379/0 diff --git a/isic/core/tests/test_isic_oauth_app.py b/isic/core/tests/test_isic_oauth_app.py index ae07edc11..e3834504b 100644 --- a/isic/core/tests/test_isic_oauth_app.py +++ b/isic/core/tests/test_isic_oauth_app.py @@ -86,6 +86,7 @@ def is_staff_view(request): yield urlpatterns.remove(urlpattern) + NinjaAPI._registry.remove(request.function.__name__) def get_bearer_token(user, oauth_token_factory): diff --git a/isic/core/tests/test_storage.py b/isic/core/tests/test_storage.py index 5a15c0f1e..126eef9cd 100644 --- a/isic/core/tests/test_storage.py +++ b/isic/core/tests/test_storage.py @@ -21,5 +21,8 @@ def test_expiration_time(t_now): def test_prevent_renaming(): storages["default"].save("foo", ContentFile(b"test")) - with pytest.raises(Exception, match="already exists."): - storages["default"].save("foo", ContentFile(b"test")) + try: + with pytest.raises(Exception, match="already exists."): + storages["default"].save("foo", ContentFile(b"test")) + finally: + storages["default"].delete("foo")