GitHub integration

.dockerignore

@@ -5,3 +5,5 @@ build
 incoming
 docker
 *.zip
+logs
+backups

Makefile

@@ -9,7 +9,7 @@ SECRETS_DIR=${BUILD_DIR}/secrets
 DB_PASSWORD_PATH=${SECRETS_DIR}/db_password
 PGPASS_PATH=${SECRETS_DIR}/.pgpass
 SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key
-EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key
+EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key github_integration_app_private_key github_integration_app_webhook_secret
 GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH)
 
 ENVREPLACE := deploy/scripts/envreplace
@@ -32,7 +32,7 @@ include .env
 
 .PHONY: build
 build: docker-compose.yml secrets $(DOCKER_SHARED_DIR)
-	docker compose build --pull -q
+	docker compose --progress=plain build --pull --parallel
 
 $(BORG_REPO_PATH):
 	wget -c ${BORG_REPO_URL} -P ${BUILD_DIR}
@@ -136,7 +136,7 @@ clean_deploy: clean
 
 .PHONY: test
 test: build
-	docker compose run --rm server /code/deploy/test.sh
+	docker compose run --rm server /code/deploy/test.sh $(TEST_ARGS)
 
 # e2e testing setup
 
@@ -157,3 +157,7 @@ e2e: docker-compose.yml secrets $(DOCKER_SHARED_DIR) $(E2E_REPO_PATH)
 	docker compose -f docker-compose.yml -f e2e.yml exec server bash -c "\
 		inv borg.restore --force && \
 		inv prepare"
+
+.PHONY: gen-secret
+gen-secret:
+	docker compose run --rm server python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())"

base.yml

@@ -67,6 +67,8 @@ services:
       - django_secret_key
       - github_client_secret
       - orcid_client_secret
+      - github_integration_app_private_key
+      - github_integration_app_webhook_secret
       - hcaptcha_secret
       - mail_api_key
       - youtube_api_key
@@ -99,6 +101,10 @@ secrets:
     file: ./build/secrets/django_secret_key
   github_client_secret:
     file: ./build/secrets/github_client_secret
+  github_integration_app_private_key:
+    file: ./build/secrets/github_integration_app_private_key
+  github_integration_app_webhook_secret:
+    file: ./build/secrets/github_integration_app_webhook_secret
   hcaptcha_secret:
     file: ./build/secrets/hcaptcha_secret
   mail_api_key:

deploy/conf/.env.template

@@ -45,6 +45,12 @@ DATACITE_DRY_RUN="true" # allowed values: "true" or "false"
 # youtube api settings
 YOUTUBE_CHANNEL_ID=
 
+# github integration app
+GITHUB_INTEGRATION_APP_ID=
+GITHUB_INTEGRATION_APP_NAME=
+GITHUB_INTEGRATION_APP_INSTALLATION_ID=
+GITHUB_MODEL_LIBRARY_ORG_NAME=
+
 # test
 TEST_USER_ID=10000000
 TEST_USERNAME=__test_user__

django/Dockerfile

@@ -10,12 +10,13 @@ ENV PATH "${VIRTUAL_ENV}/bin:$PATH"
 RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 RUN --mount=type=cache,target=/var/lib/apt,sharing=locked \
     --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    DEBIAN_FRONTEND=noninteractive \
     sed -i "s|archive.ubuntu.com|${UBUNTU_MIRROR}|" /etc/apt/sources.list \
-        && apt-get update \
-        && apt-get install -y postgresql-common --no-install-recommends \
+        && apt-get -q update \
+        && apt-get -q install -y postgresql-common --no-install-recommends \
         && yes | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh \
-        && apt-get update \
-        && apt-get install -y --no-install-recommends \
+        && apt-get -q update \
+        && apt-get -q install -y --no-install-recommends \
         autopostgresqlbackup \
         binutils \
         borgbackup \

django/core/models.py

@@ -457,7 +457,6 @@ def orcid_url(self):
     def avatar_url(self):
         if self.picture:
             return self.picture.get_rendition("fill-150x150").url
-        return None
 
     @property
     def github_url(self):
@@ -466,11 +465,16 @@ def github_url(self):
         """
         return self.get_social_account_profile_url("github")
 
+    @property
+    def github_username(self):
+        github_account = self.get_social_account("github")
+        if github_account:
+            return github_account.extra_data.get("login")
+
     def get_social_account_profile_url(self, provider_name):
         social_acct = self.get_social_account(provider_name)
         if social_acct:
             return social_acct.get_profile_url()
-        return None
 
     def get_social_account(self, provider_name):
         return self.user.socialaccount_set.filter(provider=provider_name).first()

django/core/settings/defaults.py

@@ -553,6 +553,20 @@ def set_environment(env: Environment):
 GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "")
 GITHUB_CLIENT_SECRET = read_secret("github_client_secret")
 
+GITHUB_INTEGRATION_APP_ID = os.getenv("GITHUB_INTEGRATION_APP_ID", "")
+GITHUB_INTEGRATION_APP_NAME = os.getenv("GITHUB_INTEGRATION_APP_NAME", "")
+GITHUB_INTEGRATION_APP_PRIVATE_KEY = read_secret("github_integration_app_private_key")
+GITHUB_INTEGRATION_APP_INSTALLATION_ID = int(
+    os.getenv("GITHUB_INTEGRATION_APP_INSTALLATION_ID") or 0
+)
+GITHUB_INTEGRATION_APP_WEBHOOK_SECRET = read_secret(
+    "github_integration_app_webhook_secret"
+)
+GITHUB_MODEL_LIBRARY_ORG_NAME = os.getenv("GITHUB_MODEL_LIBRARY_ORG_NAME", "")
+GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT = int(
+    os.getenv("GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT") or 100 * 1024 * 1024
+)
+
 TEST_BASIC_AUTH_PASSWORD = os.getenv("TEST_BASIC_AUTH_PASSWORD", "test password")
 TEST_USER_ID = os.getenv("TEST_USER_ID", 1000000)
 TEST_USERNAME = os.getenv("TEST_USERNAME", "__test_user__")

django/core/tests/base.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
+from pathlib import Path
 import logging
-import os
 import shlex
 import shutil
 import subprocess
@@ -179,13 +179,21 @@ def initialize_test_shared_folders():
         settings.BACKUP_ROOT,
         settings.MEDIA_ROOT,
     ]:
-        os.makedirs(d, exist_ok=True)
+        Path(d).mkdir(parents=True, exist_ok=True)
 
     subprocess.run(
         shlex.split("borg init --encryption=none {}".format(settings.BORG_ROOT)),
         check=True,
     )
 
 
+def clear_test_shared_folder(dir=settings.REPOSITORY_ROOT):
+    for fs in Path(dir).iterdir():
+        if fs.is_dir():
+            shutil.rmtree(fs, ignore_errors=True)
+        elif fs.is_file():
+            fs.unlink()
+
+
 def destroy_test_shared_folders():
-    shutil.rmtree(settings.SHARE_DIR, ignore_errors=True)
+    shutil.rmtree(Path(settings.SHARE_DIR), ignore_errors=True)

django/curator/fs.py

@@ -8,6 +8,8 @@
 def fsck(queryset):
     results = OrderedDict()
     for release in queryset:
+        if release.is_imported:
+            continue
         rfsc = CodebaseReleaseFileConsistencyChecker(release)
         errors = rfsc.check()
         if errors:

django/curator/invoke_tasks/borg.py

@@ -1,5 +1,4 @@
-import os
-import pathlib
+from pathlib import Path
 import shutil
 import tempfile
 
@@ -9,13 +8,14 @@
 from . import database as db
 from core.utils import confirm
 
-DEFAULT_LIBRARY_BASENAME = os.path.basename(settings.LIBRARY_ROOT)
-DEFAULT_MEDIA_BASENAME = os.path.basename(settings.MEDIA_ROOT)
+DEFAULT_LIBRARY_BASENAME = Path(settings.LIBRARY_ROOT).name
+DEFAULT_MEDIA_BASENAME = Path(settings.MEDIA_ROOT).name
+DEFAULT_REPOSITORY_BASENAME = Path(settings.REPOSITORY_ROOT).name
 
 
 @task(aliases=["init"])
 def initialize_repo(ctx):
-    if not os.path.exists(settings.BORG_ROOT):
+    if not Path(settings.BORG_ROOT).exists():
         ctx.run(f"borg init --encryption=none {settings.BORG_ROOT}", echo=True)
 
 
@@ -46,61 +46,85 @@ def backup(ctx):
     # Borg recognizes {now} as the current timestamp
     #  http://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-placeholders
     archive = "{utcnow}"
-    library = os.path.relpath(settings.LIBRARY_ROOT, share)
-    media = os.path.relpath(settings.MEDIA_ROOT, share)
-    database = os.path.relpath(os.path.join(settings.BACKUP_ROOT, "latest"), share)
+    share_path = Path(share)
+    library_root = Path(settings.LIBRARY_ROOT)
+    media_root = Path(settings.MEDIA_ROOT)
+    repository_root = Path(settings.REPOSITORY_ROOT)
+    backup_latest_root = Path(settings.BACKUP_ROOT) / "latest"
+
+    library = library_root.relative_to(share_path)
+    media = media_root.relative_to(share_path)
+    repository = repository_root.relative_to(share_path)
+    database = backup_latest_root.relative_to(share_path)
 
     error_msgs = []
-    for p in (
-        settings.LIBRARY_ROOT,
-        settings.MEDIA_ROOT,
-        os.path.join(settings.BACKUP_ROOT, "latest"),
-    ):
-        if not os.path.exists(p):
+    for p in (library_root, media_root, repository_root, backup_latest_root):
+        if not p.exists():
             error_msgs.append(f"Path {p} does not exist.")
     if error_msgs:
         raise IOError("Create archive failed. {}".format(" ".join(error_msgs)))
 
     with ctx.cd(share):
         ctx.run(
-            f'borg create --stats --compression lz4 {repo}::"{archive}" {library} {media} {database}',
+            f'borg create --stats --compression lz4 {repo}::"{archive}" {library} {media} {repository} {database}',
             echo=True,
             env=environment(),
         )
 
 
 def delete_latest_uncompressed_backup(
-    src_library=DEFAULT_LIBRARY_BASENAME, src_media=DEFAULT_MEDIA_BASENAME
+    src_library=DEFAULT_LIBRARY_BASENAME,
+    src_media=DEFAULT_MEDIA_BASENAME,
+    src_repository=DEFAULT_REPOSITORY_BASENAME,
 ):
-    latest_dest_library = os.path.join(settings.PREVIOUS_SHARE_ROOT, src_library)
-    latest_dest_media = os.path.join(settings.PREVIOUS_SHARE_ROOT, src_media)
+    previous = Path(settings.PREVIOUS_SHARE_ROOT)
+    latest_dest_library = previous / src_library
+    latest_dest_media = previous / src_media
+    latest_dest_repository = previous / src_repository
 
     shutil.rmtree(latest_dest_library, ignore_errors=True)
     shutil.rmtree(latest_dest_media, ignore_errors=True)
+    shutil.rmtree(latest_dest_repository, ignore_errors=True)
 
 
 def rotate_library_and_media_files(
     working_directory,
     src_library=DEFAULT_LIBRARY_BASENAME,
     src_media=DEFAULT_MEDIA_BASENAME,
+    src_repository=DEFAULT_REPOSITORY_BASENAME,
 ):
     """
-    Rotate the current library and media files
+    Rotate the current library, media, and repository files
 
-    Current library and media files are moved to the '.latest' folder in case of problems during the restore process.
+    Current library, media, and repository files are moved to the '.latest' folder in case of problems during the restore process.
     Files in .latest can be deleted if the restore was successful
     """
-    print("rotating library and media files")
-    delete_latest_uncompressed_backup(src_library=src_library, src_media=src_media)
-
-    os.makedirs(settings.PREVIOUS_SHARE_ROOT, exist_ok=True)
-    if os.path.exists(settings.LIBRARY_ROOT):
-        shutil.move(settings.LIBRARY_ROOT, settings.PREVIOUS_SHARE_ROOT)
-    if os.path.exists(settings.MEDIA_ROOT):
-        shutil.move(settings.MEDIA_ROOT, settings.PREVIOUS_SHARE_ROOT)
+    print("rotating library, media, and repository files")
+    delete_latest_uncompressed_backup(
+        src_library=src_library, src_media=src_media, src_repository=src_repository
+    )
 
-    shutil.move(os.path.join(working_directory, src_library), settings.SHARE_DIR)
-    shutil.move(os.path.join(working_directory, src_media), settings.SHARE_DIR)
+    previous = Path(settings.PREVIOUS_SHARE_ROOT)
+    share_root = Path(settings.SHARE_DIR)
+    library_root = Path(settings.LIBRARY_ROOT)
+    media_root = Path(settings.MEDIA_ROOT)
+    repository_root = Path(settings.REPOSITORY_ROOT)
+    working_root = Path(working_directory)
+
+    previous.mkdir(exist_ok=True)
+    if library_root.exists():
+        shutil.move(library_root, previous)
+    if media_root.exists():
+        shutil.move(media_root, previous)
+    if repository_root.exists():
+        shutil.move(repository_root, previous)
+
+    shutil.move(working_root / src_library, share_root)
+    shutil.move(working_root / src_media, share_root)
+    # repository dir may not exist in older backups
+    src_repository_path = working_root / src_repository
+    if src_repository_path.exists():
+        shutil.move(src_repository_path, share_root)
 
 
 def environment():
@@ -115,11 +139,8 @@ def _restore_files(working_directory):
 
 
 def _restore_database(ctx, working_directory, target_database):
-    dumpfile_dir = pathlib.Path(
-        os.path.join(
-            working_directory, os.path.basename(settings.BACKUP_ROOT), "latest"
-        )
-    )
+    backup_root_name = Path(settings.BACKUP_ROOT).name
+    dumpfile_dir = Path(working_directory) / backup_root_name / "latest"
     if not dumpfile_dir.exists():
         raise IOError("dumpfile_dir {} not found".format(dumpfile_dir))
     dumpfile = str(list(dumpfile_dir.glob("comsesnet*"))[0])

django/curator/tests/test_dump_restore.py

@@ -19,7 +19,7 @@
 from core.tests.base import EventFactory, JobFactory
 from library.fs import import_archive
 from library.models import Codebase
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def setUp(self):
         fs_api = self.release.get_fs_api()
         import_archive(
             codebase_release=self.release,
-            nested_code_folder_name="library/tests/archives/nestedcode",
+            nested_code_folder_name=TEST_SAMPLES_DIR / "archives" / "nestedcode",
             fs_api=fs_api,
         )
 

django/deploy/test.sh

@@ -8,4 +8,10 @@ initdb() {
     env DJANGO_SETTINGS_MODULE="core.settings.test" invoke db.init
 }
 initdb
-exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke collectstatic test --tests="$@" --coverage
+
+if [ "$#" -gt 0 ]; then
+    TEST_SELECTOR="$*"
+    exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke collectstatic test --tests="$TEST_SELECTOR" --coverage
+fi
+
+exec env DJANGO_SETTINGS_MODULE="core.settings.test" invoke collectstatic test --coverage