From c6e5b56f4aff898f3bbc2214631bd09d9697f91c Mon Sep 17 00:00:00 2001 From: sgfost Date: Thu, 9 Jan 2025 14:18:07 -0700 Subject: [PATCH 01/66] feat: add model for tracking git mirror of a codebase --- ...3_codebasegitmirror_codebase_git_mirror.py | 69 ++++++++++++++++ django/library/models.py | 79 +++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py diff --git a/django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py b/django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py new file mode 100644 index 000000000..3d8510af1 --- /dev/null +++ b/django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py @@ -0,0 +1,69 @@ +# Generated by Django 4.2.17 on 2025-01-09 20:57 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("library", "0032_license_text_codemeta_snapshot"), + ] + + operations = [ + migrations.CreateModel( + name="CodebaseGitMirror", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("last_modified", models.DateTimeField(auto_now=True)), + ("repository_name", models.CharField(max_length=100, unique=True)), + ( + "remote_url", + models.URLField( + blank=True, help_text="URL of mirrored remote repository" + ), + ), + ("last_local_update", models.DateTimeField(blank=True, null=True)), + ("last_remote_update", models.DateTimeField(blank=True, null=True)), + ( + "user_access_token", + models.CharField(blank=True, max_length=200, null=True), + ), + ( + "organization_login", + models.CharField(blank=True, max_length=100, null=True), + ), + ( + "local_releases", + models.ManyToManyField( + related_name="+", to="library.codebaserelease" + ), + ), + ( + "remote_releases", + models.ManyToManyField( + related_name="+", to="library.codebaserelease" + ), + ), + ], + ), + migrations.AddField( + model_name="codebase", + name="git_mirror", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="codebase", + to="library.codebasegitmirror", + ), + ), + ] diff --git a/django/library/models.py b/django/library/models.py index e91846749..51da5fbe3 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -616,6 +616,64 @@ def updated_after(self, start_date, end_date=None, **kwargs): return new_codebases, updated_codebases, releases +class CodebaseGitMirror(models.Model): + """ + Keeps track of a git repository and its GitHub remote that were created + from a Codebase using the mirror (read-only archiving) workflow + """ + + # is_active = models.BooleanField(default=True) + date_created = models.DateTimeField(auto_now_add=True) + last_modified = models.DateTimeField(auto_now=True) + repository_name = models.CharField(max_length=100, unique=True) + remote_url = models.URLField( + blank=True, + help_text=_("URL of mirrored remote repository"), + ) + # keep track of timestamp and releases that have been mirrored locally + last_local_update = models.DateTimeField(null=True, blank=True) + local_releases = models.ManyToManyField("CodebaseRelease", related_name="+") + # keep track of timestamp and releases that have been synced to the remote + last_remote_update = models.DateTimeField(null=True, blank=True) + remote_releases = models.ManyToManyField("CodebaseRelease", related_name="+") + user_access_token = models.CharField(max_length=200, null=True, blank=True) + organization_login = models.CharField(max_length=100, null=True, blank=True) + + @property + def latest_local_release(self): + return max(self.local_releases.all(), key=lambda r: Version(r.version_number)) + + @property + def latest_remote_release(self): + return max(self.remote_releases.all(), key=lambda r: Version(r.version_number)) + + @property + def unmirrored_local_releases(self): + return self.codebase.public_releases().exclude( + id__in=self.local_releases.values_list("id", flat=True) + ) + + @property + def unmirrored_remote_releases(self): + return self.local_releases.exclude( + id__in=self.remote_releases.values_list("id", flat=True) + ) + + def update_local_releases(self, new_releases: models.QuerySet | list): + if self.local_releases.exists(): + self.local_releases.add(*new_releases) + else: + self.local_releases.set(new_releases) + self.last_local_update = timezone.now() + self.save() + + def update_remote_releases(self): + releases = self.local_releases.all() + self.remote_releases.set(releases) + self.last_remote_update = timezone.now() + self.save() + + @add_to_comses_permission_whitelist class Codebase(index.Indexed, ModeratedContent, ClusterableModel): """ @@ -655,6 +713,13 @@ class Codebase(index.Indexed, ModeratedContent, ClusterableModel): on_delete=models.SET_NULL, ) + git_mirror = models.OneToOneField( + "CodebaseGitMirror", + null=True, + related_name="codebase", + on_delete=models.SET_NULL, + ) + repository_url = models.URLField( blank=True, help_text=_( @@ -833,6 +898,16 @@ def base_library_dir(self): def base_git_dir(self): return pathlib.Path(settings.REPOSITORY_ROOT, str(self.uuid)) + def create_git_mirror(self, repository_name, **kwargs): + if not self.git_mirror: + git_mirror = CodebaseGitMirror.objects.create( + repository_name=repository_name, + **kwargs, + ) + self.git_mirror = git_mirror + self.save() + return self.git_mirror + @property def publication_year(self): return ( @@ -1911,6 +1986,10 @@ def publish(self): schedule_mint_public_doi( self.id, dry_run=settings.DEPLOY_ENVIRONMENT.is_development ) + if self.codebase.git_mirror: + from .tasks import update_mirrored_codebase + + transaction.on_commit(lambda: update_mirrored_codebase(self.codebase.id)) def _publish(self): if not self.live: From 43147940588b5a1829f5dd4d639770f69df8de05 Mon Sep 17 00:00:00 2001 From: sgfost Date: Thu, 9 Jan 2025 14:26:45 -0700 Subject: [PATCH 02/66] feat: add codebase git repository fs api this API is responsible for managing a local git repository mirror for a comses codebase. PUBLIC release archives are commits/tags in the history. Release branches are created for each release and only added to if there is an update to metadata `build()` and `append_releases()` are the two main API methods which construct (or rebuild) a git repo and add new releases to the repo, respectively `update_release_branch()` will add a new commit containing changes to a release branch (and update main if they point to the same thing). This will mainly be used for updating metadata --- django/core/models.py | 8 + django/core/tests/base.py | 8 + django/curator/tests/test_dump_restore.py | 4 +- django/library/fs.py | 342 +++++++++++++++++- django/library/tests/base.py | 23 +- .../tests/{ => samples}/archives/.gitignore | 0 .../tests/{ => samples}/archives/invalid.zip | 0 .../archives/nestedcode/.DS_store} | 0 .../archives/nestedcode/.svn/svn_files_here | 0 .../archives/nestedcode/README.md | 0 .../archives/nestedcode/src/ex.py | 0 .../animals-model/1.0.0/code/model.py | 1 + .../animals-model/1.0.0/data/input.csv | 2 + .../animals-model/1.0.0/docs/README.txt | 3 + .../animals-model/1.0.0/results/analysis.txt | 1 + .../animals-model/2.0.0/code/animals/cow.py | 3 + .../animals-model/2.0.0/code/animals/horse.py | 3 + .../animals-model/2.0.0/code/animals/sheep.py | 3 + .../animals-model/2.0.0/code/model.py | 1 + .../animals-model/2.0.0/data/input.csv | 2 + .../animals-model/2.0.0/docs/README.md | 5 + django/library/tests/test_fs.py | 174 ++++++++- django/requirements.txt | 2 + 23 files changed, 568 insertions(+), 17 deletions(-) rename django/library/tests/{ => samples}/archives/.gitignore (100%) rename django/library/tests/{ => samples}/archives/invalid.zip (100%) rename django/library/tests/{archives/nestedcode/.DS_Store => samples/archives/nestedcode/.DS_store} (100%) rename django/library/tests/{ => samples}/archives/nestedcode/.svn/svn_files_here (100%) rename django/library/tests/{ => samples}/archives/nestedcode/README.md (100%) rename django/library/tests/{ => samples}/archives/nestedcode/src/ex.py (100%) create mode 100644 django/library/tests/samples/releases/animals-model/1.0.0/code/model.py create mode 100644 django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv create mode 100644 django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt create mode 100644 django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt create mode 100644 django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py create mode 100644 django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py create mode 100644 django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py create mode 100644 django/library/tests/samples/releases/animals-model/2.0.0/code/model.py create mode 100644 django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv create mode 100644 django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md diff --git a/django/core/models.py b/django/core/models.py index 2c394d0fe..be945edc2 100644 --- a/django/core/models.py +++ b/django/core/models.py @@ -466,6 +466,14 @@ def github_url(self): """ return self.get_social_account_profile_url("github") + @property + def github_username(self): + github_account = self.get_social_account("github") + if github_account: + return github_account.extra_data.get("login") + else: + return None + def get_social_account_profile_url(self, provider_name): social_acct = self.get_social_account(provider_name) if social_acct: diff --git a/django/core/tests/base.py b/django/core/tests/base.py index 7d8852a21..1eb58bf09 100644 --- a/django/core/tests/base.py +++ b/django/core/tests/base.py @@ -187,5 +187,13 @@ def initialize_test_shared_folders(): ) +def clear_test_shared_folder(dir=settings.REPOSITORY_ROOT): + for fs in os.scandir(dir): + if fs.is_dir(): + shutil.rmtree(os.path.join(dir, fs.name), ignore_errors=True) + elif fs.is_file(): + os.remove(os.path.join(dir, fs.name)) + + def destroy_test_shared_folders(): shutil.rmtree(settings.SHARE_DIR, ignore_errors=True) diff --git a/django/curator/tests/test_dump_restore.py b/django/curator/tests/test_dump_restore.py index 0d232ebb9..b7b71a39b 100644 --- a/django/curator/tests/test_dump_restore.py +++ b/django/curator/tests/test_dump_restore.py @@ -19,7 +19,7 @@ from core.tests.base import EventFactory, JobFactory from library.fs import import_archive from library.models import Codebase -from library.tests.base import CodebaseFactory +from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR logger = logging.getLogger(__name__) @@ -51,7 +51,7 @@ def setUp(self): fs_api = self.release.get_fs_api() import_archive( codebase_release=self.release, - nested_code_folder_name="library/tests/archives/nestedcode", + nested_code_folder_name=TEST_SAMPLES_DIR / "archives" / "nestedcode", fs_api=fs_api, ) diff --git a/django/library/fs.py b/django/library/fs.py index bbd13f6b5..9eee5a731 100644 --- a/django/library/fs.py +++ b/django/library/fs.py @@ -1,3 +1,5 @@ +import json +import yaml import logging import mimetypes import os @@ -5,11 +7,15 @@ import shutil import tarfile import zipfile +import filecmp +from contextlib import contextmanager +from packaging.version import Version from enum import Enum from functools import total_ordering from pathlib import Path from tempfile import TemporaryDirectory -from typing import Optional +from typing import Callable, Optional +from git import Actor, GitCommandError, InvalidGitRepositoryError, Repo import bagit import rarfile @@ -17,6 +23,7 @@ from django.core.files.storage import FileSystemStorage from django.core.files.uploadedfile import File from django.urls import reverse +from django.utils import timezone from rest_framework.exceptions import ValidationError from core import fs @@ -794,6 +801,339 @@ def rebuild(self, metadata_only=False) -> MessageGroup: return msgs +class CodebaseGitRepositoryApi: + """ + Manage a (local) git repository mirror of a codebase + """ + + FILE_SIZE_LIMIT = settings.GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT + MEGABYTE = 1024 * 1024 + FILE_SIZE_LIMIT_MB = FILE_SIZE_LIMIT / MEGABYTE + DEFAULT_BRANCH_NAME = "main" + RELEASE_BRANCH_PREFIX = "release/" + + def __init__(self, codebase): + self.codebase = codebase + self.mirror = codebase.git_mirror + if not self.mirror: + raise ValueError("Codebase must have a git_mirror") + self.repo_dir = Path(self.codebase.base_git_dir, str(self.repo_name)).absolute() + + @property + def repo_name(self): + return self.mirror.repository_name + + @property + def committer(self): + return Actor("CoMSES Net", settings.EDITOR_EMAIL) + + @property + def author(self): + profile = self.codebase.submitter.member_profile + author_email = ( + f"{profile.github_username}@users.noreply.github.com" + if profile.github_username + else profile.email + ) + return Actor(profile.name, author_email) + + def get_release_branch_name(self, release): + return f"{self.RELEASE_BRANCH_PREFIX}{release.version_number}" + + @classmethod + def check_file_sizes(cls, codebase): + releases = codebase.ordered_releases_list() + for release in releases: + release_fs_api = release.get_fs_api() + sip_storage = release_fs_api.get_sip_storage() + for file in sip_storage.list(absolute=True): + if file.stat().st_size > cls.FILE_SIZE_LIMIT: + file_size_mb = file.stat().st_size / cls.MEGABYTE + raise ValidationError( + f"File {file} is too large ({file_size_mb}MB), individual files must be under {cls.FILE_SIZE_LIMIT_MB}MB" + ) + + @contextmanager + def use_temporary_repo(self, from_existing=False): + """ + context manager that allows for 'atomic' operations on the git repository + by creating a temporary copy and copying it back after the block is executed + """ + original_repo_dir = self.repo_dir + with TemporaryDirectory() as tmpdir: + self.repo_dir = Path(tmpdir) + if from_existing: + shutil.copytree(original_repo_dir, self.repo_dir, dirs_exist_ok=True) + self.initialize(should_exist=True) + yield + if original_repo_dir.exists(): + shutil.rmtree(original_repo_dir) + shutil.copytree(self.repo_dir, original_repo_dir, dirs_exist_ok=True) + self.repo_dir = original_repo_dir + + def initialize(self, should_exist=False): + """ + initialize the git repository or connect to an existing one + + :param should_exist: if True, raise an error if the repository does not exist + """ + if not self.repo_dir.exists(): + if should_exist: + raise RuntimeError(f"Repository {self.repo_dir} does not exist") + self.repo_dir.mkdir(parents=True) + try: + self.repo = Repo(self.repo_dir) + except InvalidGitRepositoryError: + if should_exist: + raise RuntimeError(f"Repository {self.repo_dir} does not exist") + self.repo = Repo.init( + self.repo_dir, initial_branch=self.DEFAULT_BRANCH_NAME + ) + except Exception as e: + logger.exception(e) + raise RuntimeError(f"Failed to initialize git repository") + + def checkout_main(self): + self.repo.git.checkout(self.DEFAULT_BRANCH_NAME) + + def clear_existing_files(self): + """ + clear any existing files in the working tree (tracked or untracked) besides .git + """ + for item in self.repo_dir.iterdir(): + if item.name != ".git": + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() + self.repo.index.remove( + [str(item.relative_to(self.repo_dir))], + working_tree=True, + r=True, + ) + + def add_release_files(self, release): + """ + copy over submission package files for a release to the working tree of the git repo + starting from a clean directory by removing all files except .git/ + """ + release_fs_api: CodebaseReleaseFsApi = release.get_fs_api() + sip_storage = release_fs_api.get_sip_storage() + self.clear_existing_files() + # copy over files from the sip storage and add to the index + # FIXME: consider moving this copy all operation to the CodebaseReleaseStorage class + for file in sip_storage.list(absolute=True): + rel_path = file.relative_to(sip_storage.location) + dest_path = self.repo_dir / rel_path + dest_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(file, dest_path) + self.repo.index.add([str(rel_path)]) + + def add_readme(self, release): + """ + add a readme file to the repository root. If one already exists somewhere, move it. + Otherwise, generate one from a template + """ + release_fs_api: CodebaseReleaseFsApi = release.get_fs_api() + sip_storage = release_fs_api.get_sip_storage() + readme_pattern = re.compile( + r"(?i)^readme(?:\.(?:markdown|mdown|mkdn|md|textile|rdoc|org|creole|mediawiki|wiki|rst|asciidoc|adoc|asc|pod|txt))?$" + ) + for file in sip_storage.list(absolute=True): + # check for an existing readme and duplicate it to the repo root + # for github to recognize. Otherwise, we'll generate one later + if readme_pattern.match(file.name): + shutil.copy(file, self.repo_dir / file.name) + self.repo.index.add([file.name]) + return + readme_content = f"# {self.codebase.title}\n\n{self.codebase.description.raw}\n" + self._add_single_file("README.md", readme_content) + + def _add_single_file(self, filename, content: str, overwrite=False): + dest_path = self.repo_dir / filename + if not dest_path.exists() or overwrite: + with dest_path.open("w") as f: + f.write(content) + self.repo.index.add([filename]) + + def commit_release(self, release, tag=True): + """ + commit the the release and tag it, should only be called after adding all necessary files + """ + # make sure the commit goes to main, then create the release branch later + # unless this is the first commit + if self.DEFAULT_BRANCH_NAME in self.repo.heads: + self.checkout_main() + commit_msg = ( + f"Release {release.version_number}\n\n{release.release_notes.raw}\n" + ) + for rc in release.coauthor_release_contributors: + contributor = rc.contributor + email = "" + # try to use the co-author's github account email, otherwise just leave it blank + if contributor.user and contributor.user.member_profile.github_username: + email = f"{contributor.user.member_profile.github_username}@users.noreply.github.com" + commit_msg += f"\nCo-authored-by: {contributor.name} <{email}>" + commit = self.repo.index.commit( + message=commit_msg, + committer=self.committer, + author=self.author, + author_date=release.last_published_on, + ) + if tag: + self.repo.create_tag(f"{release.version_number}") + return commit + + def create_release_branch(self, release, commit): + """ + create a new branch for the release + """ + release_branch_name = self.get_release_branch_name(release) + self.repo.create_head(release_branch_name, commit) + return release_branch_name + + def update_release_branch(self, release) -> Repo | None: + """ + update a release branch with new metadata, merging back into main (fast-forward) + if it is the latest release + + this ONLY updates metadata files and does not add + changes to the code, docs, etc. as it is assumed that any synced releases are published + and frozen + + returns None if no changes were made, otherwise returns the updated repo + """ + with self.use_temporary_repo(from_existing=True): + self.initialize(should_exist=True) + release_branch_name = self.get_release_branch_name(release) + # determine whether this is the latest release (i.e. points to the + # same thing as main) and should merge back into main + release_branch = self.repo.heads[release_branch_name] + main_branch = self.repo.heads[self.DEFAULT_BRANCH_NAME] + merge_into_main = (main_branch.commit == release_branch.commit) and ( + main_branch.commit == self.repo.head.commit + ) + + self.repo.git.checkout(release_branch_name) + self.add_release_files(release) + self.add_readme(release) + + # check for changes before committing + if not self.repo.is_dirty(): + return None + + commit_msg = f"Update metadata for release {release.version_number}" + self.repo.index.commit( + message=commit_msg, + committer=self.committer, + author=self.author, + author_date=timezone.now(), + ) + if merge_into_main: + self.checkout_main() + try: + self.repo.git.merge("--ff-only", release_branch_name) + except Exception as e: + logger.error( + f"Unexpected divergence when trying to merge {release_branch_name} into {self.DEFAULT_BRANCH_NAME}: {e}" + ) + self.checkout_main() + + return Repo(self.repo_dir) + + def append_releases(self, releases=None) -> Repo: + """ + add new releases to the git repository. + releases must be newer/higher than the latest mirrored release so that they can be added on top + + this should only be used if no releases have been removed or otherwise modified since these require + rewriting history and this method strictly appends new releases + + :param releases: list of releases to append, if None, all unmirrored releases will be appended + """ + self.check_file_sizes(self.codebase) + if not releases: + releases = self.mirror.unmirrored_local_releases + if not releases: + # nothing to do, return the existing repo + return Repo(self.repo_dir) + with self.use_temporary_repo(from_existing=True): + # make sure the releases are higher than the latest mirrored release + if not all( + Version(release.version_number) + > Version(self.mirror.latest_local_release.version_number) + for release in releases + ): + raise ValueError( + "Releases must be higher than the latest mirrored release to append" + ) + # make sure the releases are ordered by version number + releases = sorted(releases, key=lambda r: Version(r.version_number)) + # append releases to the git repo by adding files, committing, and creating a branch + for release in releases: + self.add_release_files(release) + self.add_readme(release) + commit = self.commit_release(release) + self.create_release_branch(release, commit) + self.checkout_main() + # record newly mirrored releases and update timestamp + self.mirror.update_local_releases(releases) + return Repo(self.repo_dir) + + def build(self) -> Repo: + """ + builds or rebuilds the git repository from codebase releases + + this will create an entirely new repository and should only be used if we are creating the + mirror for the first time or need to rebuild the entire history + """ + self.check_file_sizes(self.codebase) + releases = self.codebase.ordered_releases_list() + if not releases: + raise ValidationError("Must have at least one public release to build from") + with self.use_temporary_repo(): + self.initialize() + for release in releases: + self.add_release_files(release) + self.add_readme(release) + commit = self.commit_release(release) + self.create_release_branch(release, commit) + self.checkout_main() + # record mirrored releases and update timestamp + self.mirror.update_local_releases(releases) + return Repo(self.repo_dir) + + def update_or_build(self) -> Repo: + if self.repo_dir.exists() and self.repo_dir.joinpath(".git").exists(): + return self.append_releases() + else: + return self.build() + + def dirs_equal(self, dir1: Path, dir2: Path, ignore=[".git"]): + """ + check if two directories are equal by recursively comparing their contents + excluding the files in the ignore list (default is just .git) + + this will likely go unused in favor of a more efficient method for checking if a + release mirror (commit) is up to date + """ + dir1 = Path(dir1) + dir2 = Path(dir2) + comparison = filecmp.dircmp(dir1, dir2, ignore=ignore) + if ( + comparison.left_only + or comparison.right_only + or comparison.diff_files + or comparison.funny_files + ): + return False + else: + for subdir in comparison.common_dirs: + if not self.dirs_equal(dir1 / subdir, dir2 / subdir): + return False + return True + + class ArchiveExtractor: def __init__(self, sip_storage: CodebaseReleaseSipStorage): self.sip_storage = sip_storage diff --git a/django/library/tests/base.py b/django/library/tests/base.py index 001478b69..2105fc4cd 100644 --- a/django/library/tests/base.py +++ b/django/library/tests/base.py @@ -1,5 +1,6 @@ import io import logging +from pathlib import Path import random from uuid import UUID @@ -19,6 +20,9 @@ ) from library.serializers import CodebaseSerializer +TEST_SAMPLES_DIR = Path("library/tests/samples") + + logger = logging.getLogger(__name__) @@ -179,7 +183,7 @@ class ReleaseSetup: PROGRAMMING_LANGUAGES = ["Python", "TypeScript"] @classmethod - def setUpPublishableDraftRelease(cls, codebase): + def setUpPublishableDraftRelease(cls, codebase, with_files=True): draft_release = codebase.create_release( status=CodebaseRelease.Status.DRAFT, initialize=True, @@ -196,15 +200,14 @@ def setUpPublishableDraftRelease(cls, codebase): release_contributor_factory = ReleaseContributorFactory(draft_release) contributor = contributor_factory.create() release_contributor_factory.create(contributor) - - code_file = io.BytesIO(b"print('hello world')") - code_file.name = "some_code_file.py" - docs_file = io.BytesIO(b"# Documentation") - docs_file.name = "some_doc_file.md" - fs_api = draft_release.get_fs_api() - fs_api.add(content=code_file, category=FileCategoryDirectories.code) - fs_api.add(content=docs_file, category=FileCategoryDirectories.docs) + if with_files: + code_file = io.BytesIO(b"print('hello world')") + code_file.name = "some_code_file.py" + docs_file = io.BytesIO(b"# Documentation") + docs_file.name = "some_doc_file.md" + fs_api = draft_release.get_fs_api() + fs_api.add(content=code_file, category=FileCategoryDirectories.code) + fs_api.add(content=docs_file, category=FileCategoryDirectories.docs) draft_release.save() - return draft_release diff --git a/django/library/tests/archives/.gitignore b/django/library/tests/samples/archives/.gitignore similarity index 100% rename from django/library/tests/archives/.gitignore rename to django/library/tests/samples/archives/.gitignore diff --git a/django/library/tests/archives/invalid.zip b/django/library/tests/samples/archives/invalid.zip similarity index 100% rename from django/library/tests/archives/invalid.zip rename to django/library/tests/samples/archives/invalid.zip diff --git a/django/library/tests/archives/nestedcode/.DS_Store b/django/library/tests/samples/archives/nestedcode/.DS_store similarity index 100% rename from django/library/tests/archives/nestedcode/.DS_Store rename to django/library/tests/samples/archives/nestedcode/.DS_store diff --git a/django/library/tests/archives/nestedcode/.svn/svn_files_here b/django/library/tests/samples/archives/nestedcode/.svn/svn_files_here similarity index 100% rename from django/library/tests/archives/nestedcode/.svn/svn_files_here rename to django/library/tests/samples/archives/nestedcode/.svn/svn_files_here diff --git a/django/library/tests/archives/nestedcode/README.md b/django/library/tests/samples/archives/nestedcode/README.md similarity index 100% rename from django/library/tests/archives/nestedcode/README.md rename to django/library/tests/samples/archives/nestedcode/README.md diff --git a/django/library/tests/archives/nestedcode/src/ex.py b/django/library/tests/samples/archives/nestedcode/src/ex.py similarity index 100% rename from django/library/tests/archives/nestedcode/src/ex.py rename to django/library/tests/samples/archives/nestedcode/src/ex.py diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py b/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py new file mode 100644 index 000000000..8cde7829c --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py @@ -0,0 +1 @@ +print("hello world") diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv b/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv new file mode 100644 index 000000000..29aa17e68 --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv @@ -0,0 +1,2 @@ +horses,sheep +10,15 diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt b/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt new file mode 100644 index 000000000..c9be3ff53 --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt @@ -0,0 +1,3 @@ +instructions: + +python model.py diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt b/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt new file mode 100644 index 000000000..29fee6381 --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt @@ -0,0 +1 @@ +result tells us there are more sheep diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py new file mode 100644 index 000000000..f4963717d --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py @@ -0,0 +1,3 @@ +class Cow: + def __init__(self, name): + self.name = name diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py new file mode 100644 index 000000000..84790eaf9 --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py @@ -0,0 +1,3 @@ +class Horse: + def __init__(self, name): + self.name = name diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py new file mode 100644 index 000000000..3c575356d --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py @@ -0,0 +1,3 @@ +class Sheep: + def __init__(self, name): + self.name = name diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py new file mode 100644 index 000000000..8cde7829c --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py @@ -0,0 +1 @@ +print("hello world") diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv b/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv new file mode 100644 index 000000000..603cf0ed3 --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv @@ -0,0 +1,2 @@ +horses,sheep,cows +10,15,5 diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md b/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md new file mode 100644 index 000000000..64258c3b0 --- /dev/null +++ b/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md @@ -0,0 +1,5 @@ +# instructions: + +``` +python model.py +``` \ No newline at end of file diff --git a/django/library/tests/test_fs.py b/django/library/tests/test_fs.py index 5bcd21f9a..d7923db8e 100644 --- a/django/library/tests/test_fs.py +++ b/django/library/tests/test_fs.py @@ -1,19 +1,24 @@ from pathlib import Path - +import os +from git import Repo from django.test import TestCase +from django.conf import settings from core.tests.base import ( UserFactory, destroy_test_shared_folders, initialize_test_shared_folders, + clear_test_shared_folder, ) from library.fs import ( FileCategoryDirectories, StagingDirectories, MessageLevels, import_archive, + CodebaseGitRepositoryApi, ) -from library.tests.base import CodebaseFactory +from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR +from library.models import License import logging @@ -26,7 +31,7 @@ def setUpModule(): class ArchiveExtractorTestCase(TestCase): - nested_code_folder = Path("library/tests/archives/nestedcode") + nested_code_folder = TEST_SAMPLES_DIR / "archives" / "nestedcode" def setUp(self): self.user_factory = UserFactory() @@ -70,7 +75,7 @@ def test_zipfile_saving(self): ) def test_invalid_zipfile_saving(self): - archive_name = "library/tests/archives/invalid.zip" + archive_name = str(TEST_SAMPLES_DIR / "archives" / "invalid.zip") fs_api = self.codebase_release.get_fs_api() with open(archive_name, "rb") as f: msgs = fs_api.add( @@ -86,5 +91,166 @@ def tearDownClass(cls): cls.nested_code_folder.with_suffix(".zip").unlink(missing_ok=True) +class GitRepoApiTestCase(TestCase): + model_dir = TEST_SAMPLES_DIR / "releases" / "animals-model" + release_1_dir = model_dir / "1.0.0" + release_2_dir = model_dir / "2.0.0" + + def setUp(self): + self.user_factory = UserFactory() + self.submitter = self.user_factory.create() + self.codebase_factory = CodebaseFactory(submitter=self.submitter) + self.codebase = self.codebase_factory.create() + self.release_1 = self.codebase.create_release() + self.git_mirror = self.codebase.create_git_mirror("animals-model") + + def tearDown(self): + clear_test_shared_folder(settings.REPOSITORY_ROOT) + + def test_repo_build(self): + update_release_from_sample( + self.release_1, self.release_1_dir, version_number="1.0.0" + ) + self.release_1.publish() + public_release_count = self.codebase.public_releases().count() + self.assertEqual(public_release_count, 1) + api = CodebaseGitRepositoryApi(self.codebase) + api.build() + # check that the mirror model is updated and the repo is built + self.assertIsNotNone(self.git_mirror.last_local_update) + self.assertEqual(self.git_mirror.local_releases.count(), 1) + self.assertTrue(os.path.exists(api.repo_dir)) + # check git stuff + repo = Repo(api.repo_dir) + self.assertFalse(repo.is_dirty()) + self.assertEqual(sum(1 for _ in repo.iter_commits()), public_release_count) + self.assertEqual(len(repo.tags), public_release_count) + # check contents + self.assertTrue(os.path.exists(api.repo_dir / "codemeta.json")) + self.assertTrue(os.path.exists(api.repo_dir / "CITATION.cff")) + self.assertTrue(os.path.exists(api.repo_dir / "LICENSE")) + fs_api = self.release_1.get_fs_api() + fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code) + for category in ["code", "data", "docs"]: + self.assertTrue( + api.dirs_equal( + fs_api.sip_contents_dir / category, + api.repo_dir / category, + ) + ) + + def test_repo_append_releases(self): + update_release_from_sample( + self.release_1, self.release_1_dir, version_number="1.0.0" + ) + self.release_1.publish() + api = CodebaseGitRepositoryApi(self.codebase) + api.build() + self.release_2 = self.codebase.create_release() + update_release_from_sample( + self.release_2, self.release_2_dir, version_number="2.0.0" + ) + self.release_2.publish() + api.append_releases() + + self.assertEqual(self.git_mirror.local_releases.count(), 2) + # check git stuff + repo = Repo(api.repo_dir) + self.assertFalse(repo.is_dirty()) + public_release_count = self.codebase.public_releases().count() + self.assertEqual(sum(1 for _ in repo.iter_commits()), public_release_count) + self.assertEqual(len(repo.tags), public_release_count) + # check contents + self.assertTrue(os.path.exists(api.repo_dir / "codemeta.json")) + self.assertTrue(os.path.exists(api.repo_dir / "CITATION.cff")) + self.assertTrue(os.path.exists(api.repo_dir / "LICENSE")) + fs_api = self.release_2.get_fs_api() + fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code) + for category in ["code", "data", "docs"]: + self.assertTrue( + api.dirs_equal( + fs_api.sip_contents_dir / category, + api.repo_dir / category, + ) + ) + + def test_will_not_append_lower_version(self): + # publish 2.0.0 first and build + update_release_from_sample( + self.release_1, self.release_1_dir, version_number="1.0.0" + ) + self.release_2 = self.codebase.create_release() + update_release_from_sample( + self.release_2, self.release_2_dir, version_number="2.0.0" + ) + self.release_2.publish() + api = CodebaseGitRepositoryApi(self.codebase) + api.build() + # now publish release 1.0.0 + self.release_1.publish() + self.assertRaises(ValueError, api.append_releases) + + def test_repo_rebuild(self): + update_release_from_sample( + self.release_1, self.release_1_dir, version_number="1.0.0" + ) + self.release_1.publish() + api = CodebaseGitRepositoryApi(self.codebase) + api.build() + self.release_2 = self.codebase.create_release() + update_release_from_sample( + self.release_2, self.release_2_dir, version_number="2.0.0" + ) + self.release_2.publish() + api.build() + + self.assertEqual(self.git_mirror.local_releases.count(), 2) + # check git stuff + repo = Repo(api.repo_dir) + self.assertFalse(repo.is_dirty()) + public_release_count = self.codebase.public_releases().count() + self.assertEqual(sum(1 for _ in repo.iter_commits()), public_release_count) + self.assertEqual(len(repo.tags), public_release_count) + # check contents + self.assertTrue(os.path.exists(api.repo_dir / "codemeta.json")) + self.assertTrue(os.path.exists(api.repo_dir / "CITATION.cff")) + self.assertTrue(os.path.exists(api.repo_dir / "LICENSE")) + fs_api = self.release_2.get_fs_api() + fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code) + for category in ["code", "data", "docs"]: + self.assertTrue( + api.dirs_equal( + fs_api.sip_contents_dir / category, + api.repo_dir / category, + ) + ) + + def tearDownModule(): destroy_test_shared_folders() + + +# helpers ================================================ + + +def upload_category(fs_api, release_dir: Path, category: str): + category_path = release_dir / category + for filepath in category_path.rglob("*"): + if filepath.is_file(): + with filepath.open("rb") as f: + relpath = filepath.relative_to(category_path) + file_name = str(relpath) + fs_api.add(FileCategoryDirectories[category], content=f, name=file_name) + + +def update_release_from_sample(release, sample_dir, version_number): + release.os = "Linux" + release.programming_languages.add("Python") + release.license = License.objects.create(name="MIT") + release.release_notes = "Initial release" + release.version_number = version_number + release.save() + fs_api = release.get_fs_api() + for category in ["code", "data", "docs"]: + upload_category(fs_api, sample_dir, category) + return release diff --git a/django/requirements.txt b/django/requirements.txt index 09d4f6690..634a23812 100644 --- a/django/requirements.txt +++ b/django/requirements.txt @@ -27,6 +27,7 @@ Django==5.2.11 drf-spectacular==0.29.0 elasticsearch-dsl>=7.0.0,<8.0.0 elasticsearch>=7.0.0,<8.0.0 +gitpython==3.1.43 html2text>=2016.9.19 huey==2.6.0 jinja2==3.1.6 @@ -37,6 +38,7 @@ numpy==1.26.4 pandas==2.2.2 psycopg2-binary==2.9.11 pyjwt[crypto]>=2.4.0,<3.0.0 +PyGithub==2.5.0 pytz==2025.2 pyyaml>=6.0.1 # used for institution -> affiliation data migration From 679a1ab56a268b01106cc3f07835382c4361ce5d Mon Sep 17 00:00:00 2001 From: sgfost Date: Thu, 9 Jan 2025 14:33:50 -0700 Subject: [PATCH 03/66] feat: add github integration api and mirroring tasks the GithubApi provides access to auth and repository actions adds 3 huey (async) tasks for creating a mirror, updating a mirror, and updating metadata for a single release of a mirror --- Makefile | 2 +- base.yml | 6 + deploy/conf/.env.template | 6 + django/core/settings/defaults.py | 17 ++ django/library/github_integration.py | 253 +++++++++++++++++++++++++++ django/library/tasks.py | 94 ++++++++++ 6 files changed, 377 insertions(+), 1 deletion(-) create mode 100644 django/library/github_integration.py diff --git a/Makefile b/Makefile index 0aec8dc5f..131fce276 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ SECRETS_DIR=${BUILD_DIR}/secrets DB_PASSWORD_PATH=${SECRETS_DIR}/db_password PGPASS_PATH=${SECRETS_DIR}/.pgpass SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key -EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key +EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key github_integration_app_private_key github_integration_app_client_secret GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH) ENVREPLACE := deploy/scripts/envreplace diff --git a/base.yml b/base.yml index 616c2828f..3d5a769e0 100644 --- a/base.yml +++ b/base.yml @@ -67,6 +67,8 @@ services: - django_secret_key - github_client_secret - orcid_client_secret + - github_integration_app_private_key + - github_integration_app_client_secret - hcaptcha_secret - mail_api_key - youtube_api_key @@ -99,6 +101,10 @@ secrets: file: ./build/secrets/django_secret_key github_client_secret: file: ./build/secrets/github_client_secret + github_integration_app_private_key: + file: ./build/secrets/github_integration_app_private_key + github_integration_app_client_secret: + file: ./build/secrets/github_integration_app_client_secret hcaptcha_secret: file: ./build/secrets/hcaptcha_secret mail_api_key: diff --git a/deploy/conf/.env.template b/deploy/conf/.env.template index 8f65f1272..93fffdb85 100644 --- a/deploy/conf/.env.template +++ b/deploy/conf/.env.template @@ -45,6 +45,12 @@ DATACITE_DRY_RUN="true" # allowed values: "true" or "false" # youtube api settings YOUTUBE_CHANNEL_ID= +# github integration app +GITHUB_INTEGRATION_APP_ID= +GITHUB_INTEGRATION_APP_INSTALLATION_ID= +GITHUB_INTEGRATION_APP_CLIENT_ID= +GITHUB_MODEL_LIBRARY_ORG_NAME= + # test TEST_USER_ID=10000000 TEST_USERNAME=__test_user__ diff --git a/django/core/settings/defaults.py b/django/core/settings/defaults.py index b69678a34..ce4e252df 100644 --- a/django/core/settings/defaults.py +++ b/django/core/settings/defaults.py @@ -553,6 +553,23 @@ def set_environment(env: Environment): GITHUB_CLIENT_ID = os.getenv("GITHUB_CLIENT_ID", "") GITHUB_CLIENT_SECRET = read_secret("github_client_secret") +GITHUB_INTEGRATION_APP_ID = int(os.getenv("GITHUB_INTEGRATION_APP_ID") or 0) +GITHUB_INTEGRATION_APP_PRIVATE_KEY = read_secret("github_integration_app_private_key") +GITHUB_INTEGRATION_APP_INSTALLATION_ID = int( + os.getenv("GITHUB_INTEGRATION_APP_INSTALLATION_ID") or 0 +) +# client id and secret are only used for getting user access tokens to be able to push +# to the user's repositories. We are not re-using the regular oauth app in order to +# keep minimal permissions +GITHUB_INTEGRATION_APP_CLIENT_ID = os.getenv("GITHUB_INTEGRATION_APP_ID", "") +GITHUB_INTEGRATION_APP_CLIENT_SECRET = read_secret( + "github_integration_app_client_secret" +) +GITHUB_MODEL_LIBRARY_ORG_NAME = os.getenv("GITHUB_MODEL_LIBRARY_ORG_NAME", "") +GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT = os.getenv( + "GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT", 100 * 1024 * 1024 +) + TEST_BASIC_AUTH_PASSWORD = os.getenv("TEST_BASIC_AUTH_PASSWORD", "test password") TEST_USER_ID = os.getenv("TEST_USER_ID", 1000000) TEST_USERNAME = os.getenv("TEST_USERNAME", "__test_user__") diff --git a/django/library/github_integration.py b/django/library/github_integration.py new file mode 100644 index 000000000..722d8a6b6 --- /dev/null +++ b/django/library/github_integration.py @@ -0,0 +1,253 @@ +import re +from github import GithubIntegration, Auth, Github +from github.GithubException import UnknownObjectException +from github.Repository import Repository as GithubRepo +from git import Repo as GitRepo +from django.conf import settings +from django.core.cache import cache +from django.utils import timezone + +from .models import Codebase + +INSTALLATION_ACCESS_TOKEN_REDIS_KEY = "github_installation_access_token" + + +class GithubRepoNameValidator: + @classmethod + def validate( + cls, + repo_name: str, + username: str | None = None, + user_access_token: str | None = None, + ): + cls._validate_format(repo_name) + if username and user_access_token: + cls._check_user_repo_name_unused(repo_name, username, user_access_token) + elif username: + raise ValueError("User access token required for user repository") + else: + cls._check_org_repo_name_unused(repo_name) + + @staticmethod + def _validate_format(repo_name: str): + if not re.fullmatch(r"[A-Za-z0-9_.-]+", repo_name): + raise ValueError( + "The repository name can only contain ASCII letters, digits, and the characters ., -, and _" + ) + if not (1 <= len(repo_name) <= 100): + raise ValueError("Repository name is too long (maximum is 100 characters)") + if repo_name.endswith(".git"): + raise ValueError("Repository name cannot end with '.git'") + if "github" in repo_name: + raise ValueError("Repository name cannot contain 'github'") + + @staticmethod + def _check_user_repo_name_unused( + repo_name: str, username: str, user_access_token: str + ): + if username in repo_name: + raise ValueError( + f"Repository name cannot contain your username: '{username}'" + ) + github = Github(user_access_token) + try: + github.get_user(username).get_repo(repo_name) + raise ValueError( + f"Repository name already exists at https://github.com/{username}/{repo_name}" + ) + except UnknownObjectException: + return True + + @staticmethod + def _check_org_repo_name_unused(repo_name: str): + if settings.GITHUB_MODEL_LIBRARY_ORG_NAME in repo_name: + raise ValueError( + f"Repository name cannot contain the organization name: '{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}'" + ) + github = Github(GithubApi.get_installation_access_token()) + try: + github.get_organization(settings.GITHUB_MODEL_LIBRARY_ORG_NAME).get_repo( + repo_name + ) + raise ValueError( + f"Repository name already exists at https://github.com/{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}/{repo_name}" + ) + except UnknownObjectException: + return True + + +class GithubApi: + """Functionality for interacting with a remote Github repository + and Github API + """ + + def __init__( + self, + codebase: Codebase, + local_repo: GitRepo, + repo_name: str, + is_user_repo=False, + organization_login: str | None = None, + user_access_token: str | None = None, + private_repo=False, + ): + if is_user_repo: + raise NotImplementedError("User repositories not yet supported") + self.private_repo = private_repo + self.codebase = codebase + self.local_repo = local_repo + self.repo_name = repo_name + self.is_user_repo = is_user_repo + if is_user_repo and not organization_login: + raise ValueError("User access token required for user repository") + if not is_user_repo and not organization_login: + raise ValueError("Organization login required for org repository") + self.organization_login = organization_login + self.user_access_token = user_access_token + self._github_repo = None + + @property + def github_repo(self) -> GithubRepo: + if not self._github_repo: + try: + self._github_repo = self._get_existing_repo() + except: + raise ValueError("Github repository not created yet") + return self._github_repo + + @property + def installation_access_token(self): + return self.get_installation_access_token() + + @classmethod + def get_installation_access_token(cls): + cached_token = cache.get(INSTALLATION_ACCESS_TOKEN_REDIS_KEY) + if cached_token: + return cached_token + return cls.refresh_installation_access_token() + + @staticmethod + def refresh_installation_access_token(): + """retrieve a new installation access token for the Github app + and cache it for future use + """ + auth = Auth.AppAuth( + settings.GITHUB_INTEGRATION_APP_ID, + settings.GITHUB_INTEGRATION_APP_PRIVATE_KEY, + ) + integration = GithubIntegration(auth=auth) + installation_auth = integration.get_access_token( + settings.GITHUB_INTEGRATION_APP_INSTALLATION_ID + ) + token = installation_auth.token + seconds_until_expiration = ( + installation_auth.expires_at - timezone.now() + ).total_seconds() + # cache the token for 1 minute less than the expiration time + cache.set( + INSTALLATION_ACCESS_TOKEN_REDIS_KEY, + token, + seconds_until_expiration - 60, + ) + return token + + @staticmethod + def get_user_access_token(code: str): + # just need to link to the app install and it will go to callback with ?code=... + """return an access token for the Github user + + this token is used to authenticate requests to the Github API + to act on behalf of the user on resources they own + """ + github = Github() + app = github.get_oauth_application( + settings.GITHUB_INTEGRATION_APP_CLIENT_ID, + settings.GITHUB_INTEGRATION_APP_CLIENT_SECRET, + ) + return app.get_access_token(code).token + + def get_or_create_repo(self) -> GithubRepo: + """get or create the Github repository for a user or organization""" + try: + return self.github_repo + except: + if self.is_user_repo: + self._github_repo = self._create_user_repo() + else: + self._github_repo = self._create_org_repo() + return self._github_repo + + def push(self, local_repo: GitRepo): + """push the local git repository to the Github repository""" + if self.is_user_repo: + raise NotImplementedError("User repositories not yet supported") + else: + token = self.installation_access_token + push_url = f"https://x-access-token:{token}@github.com/{self.github_repo.full_name}.git" + self._push_to_url(local_repo, push_url) + + def create_releases(self, local_repo: GitRepo): + """create Github releases for each tag in the local repository that + does not already have a corresponding release in the remote repository""" + for tag in local_repo.tags: + try: + existing_release = self.github_repo.get_release(tag.name) + except: + existing_release = None + if not existing_release: + self.github_repo.create_git_release( + tag.name, + name=tag.name, + message=tag.commit.message, + draft=False, + prerelease=False, + ) + + def _get_existing_repo(self): + """attempt to get an existing repository for the authenticated user or organization""" + if self.is_user_repo: + github = Github(self.user_access_token) + name = github.get_user().login + return github.get_repo(f"{name}/{self.repo_name}") + else: + github = Github(self.installation_access_token) + return github.get_repo(f"{self.organization_login}/{self.repo_name}") + + def _create_user_repo(self): + """create a new repository in the user's account + + this function requires the `repo` scope for the user access token + """ + token = self.user_access_token + if not token: + raise ValueError("User access token required for creating user repository") + github = Github(token) + repo = github.get_user().create_repo( + name=self.repo_name, + description=self.codebase.description, + private=self.private_repo, + ) + return repo + + def _create_org_repo(self): + """create a new repository in the CoMSES model library organization + + this function requires the `repo` scope for the installation access token + """ + token = self.installation_access_token + github = Github(token) + org = github.get_organization(settings.GITHUB_MODEL_LIBRARY_ORG_NAME) + repo = org.create_repo( + name=self.repo_name, + description=f"Mirror of {self.codebase.permanent_url}", + private=self.private_repo, + ) + return repo + + def _push_to_url(self, local_repo: GitRepo, push_url: str): + if "origin" not in local_repo.remotes: + local_repo.create_remote("origin", push_url) + else: + local_repo.remotes["origin"].set_url(push_url) + local_repo.git.push("--all") + local_repo.git.push("--tags") diff --git a/django/library/tasks.py b/django/library/tasks.py index 0cbc93804..31e7073d5 100644 --- a/django/library/tasks.py +++ b/django/library/tasks.py @@ -1,17 +1,111 @@ from huey.contrib.djhuey import db_task, on_commit_task +from django.conf import settings + +from .models import Codebase, CodebaseRelease +from .github_integration import GithubApi +from .fs import CodebaseGitRepositoryApi import logging logger = logging.getLogger(__name__) +@db_task(retries=3, retry_delay=30) +def mirror_codebase(codebase_id: int, private_repo=False): + """asynchronous task that mirrors a codebase to a remote Github repository""" + codebase = Codebase.objects.get(id=codebase_id) + mirror = codebase.git_mirror + if not mirror: + raise ValueError("Codebase does not have a git mirror") + mirror.organization_login = settings.GITHUB_MODEL_LIBRARY_ORG_NAME + mirror.save() + + git_fs_api = CodebaseGitRepositoryApi(codebase) + local_repo = git_fs_api.update_or_build() + + gh_api = GithubApi( + codebase=codebase, + local_repo=local_repo, + repo_name=mirror.repository_name, + is_user_repo=False, + organization_login=mirror.organization_login, + user_access_token=mirror.user_access_token, + private_repo=private_repo, + ) + repo = gh_api.get_or_create_repo() + mirror.remote_url = repo.html_url + gh_api.push(local_repo) + gh_api.create_releases(local_repo) + mirror.update_remote_releases() + + +@db_task(retries=3, retry_delay=30) +def update_mirrored_codebase(codebase_id: int): + """asynchronous task that updates a mirrored codebase by pushing new releases to Github""" + codebase = Codebase.objects.get(id=codebase_id) + mirror = codebase.git_mirror + if not mirror: + raise ValueError("Codebase does not have a git mirror") + if not mirror.remote_url: + raise ValueError("Codebase git mirror does not have a remote url") + + git_fs_api = CodebaseGitRepositoryApi(codebase) + local_repo = git_fs_api.append_releases() + gh_api = GithubApi( + codebase=codebase, + local_repo=local_repo, + repo_name=mirror.repository_name, + is_user_repo=bool(mirror.user_access_token), + organization_login=mirror.organization_login, + user_access_token=mirror.user_access_token, + ) + gh_api.push(local_repo) + gh_api.create_releases(local_repo) + mirror.update_remote_releases() + + +@db_task(retries=3, retry_delay=30) +def update_mirrored_release_metadata(release_id: int): + """asynchronous task that updates a SINGLE RELEASE BRANCH with any metadata changes + that may have occurred. + + This should be called when release metadata has been changed + """ + release = CodebaseRelease.objects.get(id=release_id) + codebase = release.codebase + mirror = codebase.git_mirror + if not mirror: + raise ValueError("Codebase does not have a git mirror") + if not mirror.remote_url: + raise ValueError("Codebase git mirror does not have a remote url") + + git_fs_api = CodebaseGitRepositoryApi(codebase) + local_repo = git_fs_api.update_release_branch(release) + if local_repo: + gh_api = GithubApi( + codebase=codebase, + local_repo=local_repo, + repo_name=mirror.repository_name, + is_user_repo=bool(mirror.user_access_token), + organization_login=mirror.organization_login, + user_access_token=mirror.user_access_token, + ) + gh_api.push(local_repo) + mirror.update_remote_releases() + + @db_task(retries=1, retry_delay=30) def update_fs_release_metadata(release_id: int): from .models import CodebaseRelease release = CodebaseRelease.objects.get(id=release_id) + codebase = release.codebase fs_api = release.get_fs_api() fs_api.rebuild(metadata_only=True) + # if the release is published and the codebase has a git mirror, + # update the metadata in the git repository + if release.is_published and codebase.git_mirror and codebase.git_mirror.remote_url: + update_mirrored_release_metadata(release_id) @on_commit_task() From 7d053f7c1dfbadd45e9b3e80e21ad87bcf49b15d Mon Sep 17 00:00:00 2001 From: sgfost Date: Thu, 9 Jan 2025 14:39:27 -0700 Subject: [PATCH 04/66] feat: github integration ui and mirroring feature * /github page to describe the integration features * sidebar element on release detail page will show information about integration status for that codebase, and allow users with edit permissions to create a new mirror --- .../library/codebases/releases/retrieve.jinja | 56 +++++ .../library/github-integration-overview.jinja | 234 ++++++++++++++++++ django/library/metadata.py | 7 +- django/library/urls.py | 10 + django/library/views.py | 45 +++- frontend/src/apps/github_mirror.ts | 8 + frontend/src/components/GithubMirrorModal.vue | 93 +++++++ .../src/components/form/HoneypotField.vue | 2 +- frontend/src/composables/api/codebase.ts | 5 + frontend/src/scss/_global.scss | 32 +++ 10 files changed, 489 insertions(+), 3 deletions(-) create mode 100644 django/library/jinja2/library/github-integration-overview.jinja create mode 100644 frontend/src/apps/github_mirror.ts create mode 100644 frontend/src/components/GithubMirrorModal.vue diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index 5922f78e5..d50e518ea 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -289,6 +289,61 @@ {% endif %} {% endwith %} +{% if codebase.git_mirror or has_change_perm %} +
+
+
+
+ + + Github Integration + + beta +
+ + {% if codebase.git_mirror %} + {% if codebase.git_mirror.remote_url %} +
+ + + {{ codebase.git_mirror.repository_name }} + + + +
+ {% else %} +
+ +

+ Mirroring in progress. This may take a few seconds. +

+ Refresh this page. + +
+ {% endif %} + {% elif has_change_perm %} + {% if codebase.live %} +
+ {% else %} +
+ Only available for published models +
+ {% endif %} + {% endif %} +
+ + What is this? + +
+
+
+
+{% endif %} +
@@ -574,5 +629,6 @@ {{ vite_asset("apps/release_download.ts") }} {% if has_change_perm %} {{ vite_asset("apps/release_regenerate_share_uuid.ts") }} + {{ vite_asset("apps/github_mirror.ts") }} {% endif %} {% endblock %} diff --git a/django/library/jinja2/library/github-integration-overview.jinja b/django/library/jinja2/library/github-integration-overview.jinja new file mode 100644 index 000000000..7facb8daf --- /dev/null +++ b/django/library/jinja2/library/github-integration-overview.jinja @@ -0,0 +1,234 @@ +{% extends "base.jinja" %} + +{% block introduction %} +

Computational Model Library Github Integration

+{% endblock %} + +{% block content %} +
+
+
+
+
+
+

Mirroring

+

+ +

+
+

+ Mirroring transforms your model into a git repository and archives it on GitHub under the + + {{ github_model_library_org_name }} organization. +

+

+ Get started by clicking the + Mirror on GitHub button on + one of your models. +

+
+
+
+
+
+
+
+

Syncing (coming soon!)

+

+ +

+
+

+ Syncing will allow you to develop your model on GitHub and have any new releases + pulled back into the CoMSES Model Library. + +

+

+ Start from either a model on CoMSES or an existing GitHub repository. +

+
+
+
+
+ +
+

How It Works

+
+
+
    +
  • + +
    + Published model releases are transformed into commits + and releases on GitHub, preserving your project's history and versioning. +
    +
    • +
  • + +
    + The original submitter is set as the commit + author, maintaining attribution. +

    + {% if not user.is_authenticated %} + We will use a connected Github account or your profile information to sign commits. + You can + connect + your Github account by editing your profile if you want it to appear as the author + of the + commits. + {% elif user.member_profile.github_username %} + Since you have a connected Github account, we will use it to sign commits: + {{ user.member_profile.github_username }} <{{ user.member_profile.github_username }}@users.noreply.github.com> + {% else %} + Since you do not have a github account connected we will use your profile + information: + {{ user.member_profile.name }} <{{ user.email }}> + {% endif %} +

    +
    +
    • +
  • + +
    + Metadata is preserved by adding to the GitHub repo where possible and stored in + versioned + codemeta.json, + LICENSE, + and CITATION.cff files. +
    +
    • +
  • + +
    + Each time a new release is published on the Model Library, the GitHub repository is + updated automatically. +
    +
    • +
+
+
+
+
+
    +
  • + +
    + When enabled on a CoMSES model, the model is transformed into a git repository + with the same process as mirroring but the repository will be created under your + GitHub account. +
    +
    • +
  • + +
    + Alternatively, you can start from an existing GitHub repository and import it into + the Model Library. +
    +
    • +
  • + +
    + When you are ready to publish a new release, update the software's metadata + and create a new release on GitHub. The Model Library will automatically pull in the + new release. +
    +
    • +
+
+
+
+ +
+

Benefits

+
+
+
    +
  • + +
    + Automatically transition your model codebase to the most commonly used + version control system +
    +
    • +
  • + +
    + Provide better access to your model's source with GitHub's code browser +
    +
    • +
+
+
+
+
+
    +
  • + +
    + Collaborate with other researchers using GitHub's rich collaboration and project + management environment +
    +
    • +
  • + +
    + Take advantage of tools available as GitHub apps + and actions +
    +
    • +
+
+
+
+ +
+

Limitations

+
+
+
    +
  • + +
    + Individual files cannot exceed 100MB +
    +
    • +
  • + +
    + Mirrored repositories are read-only, though you are free to fork them +
    +
    • +
+
+
+
+
+
    +
  • + +
    + Individual files cannot exceed 100MB +
    +
    • +
  • + +
    + Required metadata must be present in order to pull in new releases from GitHub +
    +
    • +
  • + +
    + Changes cannot be made to the CoMSES Model Library record when there are conflicting + changes on GitHub +
    +
    • +
+
+
+
+
+{% endblock %} diff --git a/django/library/metadata.py b/django/library/metadata.py index 535f5f46d..40b382708 100644 --- a/django/library/metadata.py +++ b/django/library/metadata.py @@ -140,7 +140,12 @@ def _common_codebase_fields(cls, codebase) -> dict: return dict( type_="SoftwareSourceCode", name=codebase.title, - codeRepository=codebase.repository_url or None, + codeRepository=( + codebase.git_mirror.remote_url + if codebase.git_mirror + else codebase.repository_url + ) + or None, applicationCategory="Computational Model", citation=[ cls.to_textual_creative_work(text) diff --git a/django/library/urls.py b/django/library/urls.py index b4e72a705..dc6a838a9 100644 --- a/django/library/urls.py +++ b/django/library/urls.py @@ -47,6 +47,16 @@ ) urlpatterns = [ + path( + "github/", + TemplateView.as_view( + template_name="library/github-integration-overview.jinja", + extra_context={ + "github_model_library_org_name": settings.GITHUB_MODEL_LIBRARY_ORG_NAME, + }, + ), + name="github-integration-overview", + ), path( "reviews/", TemplateView.as_view(template_name="library/review/peer-review-overview.jinja"), diff --git a/django/library/views.py b/django/library/views.py index 909fb323e..da576312b 100644 --- a/django/library/views.py +++ b/django/library/views.py @@ -1,5 +1,6 @@ import json from django.forms import Form +from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin from django.core.exceptions import PermissionDenied @@ -46,6 +47,7 @@ ) from core.pagination import SmallResultSetPagination from core.serializers import RelatedMemberProfileSerializer +from .github_integration import GithubRepoNameValidator from .forms import ( PeerReviewerFeedbackReviewerForm, PeerReviewInvitationReplyForm, @@ -53,7 +55,12 @@ PeerReviewerFeedbackEditorForm, PeerReviewFilterForm, ) -from .fs import FileCategoryDirectories, StagingDirectories, MessageLevels +from .fs import ( + CodebaseGitRepositoryApi, + FileCategoryDirectories, + StagingDirectories, + MessageLevels, +) from .models import ( Codebase, CodebaseRelease, @@ -82,6 +89,7 @@ PeerReviewFeedbackEditorSerializer, PeerReviewEventLogSerializer, ) +from .tasks import mirror_codebase, update_mirrored_codebase import logging import pathlib @@ -520,6 +528,41 @@ def retrieve(self, request, *args, **kwargs): data = add_user_retrieve_perms(instance, serializer.data, request.user) return Response(data) + # @action(detail=True, methods=["post"]) + # def github_sync(self, request, *args, **kwargs): + # pass + + @action(detail=True, methods=["post"]) + def github_mirror(self, request, *args, **kwargs): + codebase = self.get_object() + if not codebase.live: + raise ValidationError("This model does not have any published releases") + if codebase.git_mirror: + raise ValidationError("This codebase is already mirrored to a GitHub repo") + CodebaseGitRepositoryApi.check_file_sizes(codebase) + repo_name = request.data.get("repo_name") + try: + GithubRepoNameValidator.validate(repo_name) + except ValueError as e: + raise ValidationError(str(e)) + codebase.create_git_mirror(repo_name) + mirror_codebase(codebase.id, private_repo=settings.DEBUG) + return Response( + status=status.HTTP_202_ACCEPTED, + data="Mirroring process started, this should take only a few seconds. Refresh this page to see the status.", + ) + + @action(detail=True, methods=["post"]) + def update_github_mirror(self, request, *args, **kwargs): + codebase = self.get_object() + if not codebase.git_mirror: + return Response( + data={"error": "This codebase is not mirrored to a GitHub repo"}, + status=status.HTTP_400_BAD_REQUEST, + ) + update_mirrored_codebase(codebase.id) + return Response(status=status.HTTP_202_ACCEPTED) + class DevelopmentCodebaseDeleteView(mixins.DestroyModelMixin, CodebaseViewSet): lookup_field = "identifier" diff --git a/frontend/src/apps/github_mirror.ts b/frontend/src/apps/github_mirror.ts new file mode 100644 index 000000000..b7633a76b --- /dev/null +++ b/frontend/src/apps/github_mirror.ts @@ -0,0 +1,8 @@ +import "vite/modulepreload-polyfill"; + +import { createApp } from "vue"; +import GithubMirrorModal from "@/components/GithubMirrorModal.vue"; +import { extractDataParams } from "@/util"; + +const props = extractDataParams("github-mirror", ["identifier", "defaultRepoName"]); +createApp(GithubMirrorModal, props).mount("#github-mirror"); diff --git a/frontend/src/components/GithubMirrorModal.vue b/frontend/src/components/GithubMirrorModal.vue new file mode 100644 index 000000000..07c671c7b --- /dev/null +++ b/frontend/src/components/GithubMirrorModal.vue @@ -0,0 +1,93 @@ + + + diff --git a/frontend/src/components/form/HoneypotField.vue b/frontend/src/components/form/HoneypotField.vue index cd07c44a2..e07d62a1b 100644 --- a/frontend/src/components/form/HoneypotField.vue +++ b/frontend/src/components/form/HoneypotField.vue @@ -36,7 +36,7 @@ import FieldHelp from "@/components/form/FieldHelp.vue"; export interface HoneypotFieldProps { name?: string; - show: boolean; + show?: boolean; } const props = withDefaults(defineProps(), { diff --git a/frontend/src/composables/api/codebase.ts b/frontend/src/composables/api/codebase.ts index e5309db98..313cc303d 100644 --- a/frontend/src/composables/api/codebase.ts +++ b/frontend/src/composables/api/codebase.ts @@ -49,6 +49,10 @@ export function useCodebaseAPI() { return del(detailUrl(identifier, ["media", "clear"])); } + async function githubMirror(identifier: string, repoName: string, options?: RequestOptions) { + return post(detailUrl(identifier, ["github_mirror"]), { repoName }, options); + } + return { ...toRefs(state), create, @@ -59,6 +63,7 @@ export function useCodebaseAPI() { mediaClear, mediaListUrl, detailUrl, + githubMirror, searchUrl: searchUrl, }; } diff --git a/frontend/src/scss/_global.scss b/frontend/src/scss/_global.scss index 7528e6e55..edf464e51 100644 --- a/frontend/src/scss/_global.scss +++ b/frontend/src/scss/_global.scss @@ -71,6 +71,22 @@ a.anchor { } } +.w-100-with-icon { + width: 100%; + position: relative; + display: inline-flex; + align-items: center; + justify-content: center; + + .icon-left { + position: absolute; + left: 0; + top: 50%; + transform: translateY(-50%); + margin-left: 0.5rem; + } +} + .btn-group-xs > .btn, .btn-xs { padding: 0.25rem 0.4rem; @@ -383,6 +399,22 @@ nav { } } +.card-bridge::before { + content: ""; + position: absolute; + top: 0; + bottom: -100px; + left: 50%; + width: 1px; + background-color: #ccc; + z-index: -1; +} + +.card-bridge .card { + position: relative; + z-index: 1; +} + .card-metadata { @extend .card; margin-top: 0.5em; From c7b9c25052cd73578521c169f79abc4388c078be Mon Sep 17 00:00:00 2001 From: sgfost Date: Fri, 24 Jan 2025 18:35:51 -0700 Subject: [PATCH 05/66] feat: rework git integration models and github/fs apis * use installation access tokens for user repos instead of user access tokens. this is a more secure workflow * add GithubIntegrationAppInstallation model for recording app installations (this will need to be created/updated using webhooks) * CodebaseGitMirror/"mirror" now refers to the local git repository * ^ can have multiple CodebaseGitRemote's which keep track of all the information needed to push to/archive from remote repositories TODO: re-implement views --- Makefile | 2 +- base.yml | 3 - deploy/conf/.env.template | 2 +- django/core/settings/defaults.py | 8 +- django/library/fs.py | 26 +-- django/library/github_integration.py | 152 ++++++++------- django/library/metadata.py | 5 +- ...3_codebasegitmirror_codebase_git_mirror.py | 69 ------- ...thubintegrationappinstallation_and_more.py | 163 ++++++++++++++++ django/library/models.py | 180 +++++++++++++----- django/library/tasks.py | 87 ++++----- django/library/tests/test_fs.py | 2 +- 12 files changed, 432 insertions(+), 267 deletions(-) delete mode 100644 django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py create mode 100644 django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py diff --git a/Makefile b/Makefile index 131fce276..728ae7cde 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ SECRETS_DIR=${BUILD_DIR}/secrets DB_PASSWORD_PATH=${SECRETS_DIR}/db_password PGPASS_PATH=${SECRETS_DIR}/.pgpass SECRET_KEY_PATH=${SECRETS_DIR}/django_secret_key -EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key github_integration_app_private_key github_integration_app_client_secret +EXT_SECRETS=hcaptcha_secret github_client_secret orcid_client_secret discourse_api_key discourse_sso_secret mail_api_key datacite_api_password youtube_api_key github_integration_app_private_key GENERATED_SECRETS=$(DB_PASSWORD_PATH) $(PGPASS_PATH) $(SECRET_KEY_PATH) ENVREPLACE := deploy/scripts/envreplace diff --git a/base.yml b/base.yml index 3d5a769e0..b9d1c88ac 100644 --- a/base.yml +++ b/base.yml @@ -68,7 +68,6 @@ services: - github_client_secret - orcid_client_secret - github_integration_app_private_key - - github_integration_app_client_secret - hcaptcha_secret - mail_api_key - youtube_api_key @@ -103,8 +102,6 @@ secrets: file: ./build/secrets/github_client_secret github_integration_app_private_key: file: ./build/secrets/github_integration_app_private_key - github_integration_app_client_secret: - file: ./build/secrets/github_integration_app_client_secret hcaptcha_secret: file: ./build/secrets/hcaptcha_secret mail_api_key: diff --git a/deploy/conf/.env.template b/deploy/conf/.env.template index 93fffdb85..deee75f5e 100644 --- a/deploy/conf/.env.template +++ b/deploy/conf/.env.template @@ -47,8 +47,8 @@ YOUTUBE_CHANNEL_ID= # github integration app GITHUB_INTEGRATION_APP_ID= +GITHUB_INTEGRATION_APP_NAME= GITHUB_INTEGRATION_APP_INSTALLATION_ID= -GITHUB_INTEGRATION_APP_CLIENT_ID= GITHUB_MODEL_LIBRARY_ORG_NAME= # test diff --git a/django/core/settings/defaults.py b/django/core/settings/defaults.py index ce4e252df..be8fcc60c 100644 --- a/django/core/settings/defaults.py +++ b/django/core/settings/defaults.py @@ -554,17 +554,11 @@ def set_environment(env: Environment): GITHUB_CLIENT_SECRET = read_secret("github_client_secret") GITHUB_INTEGRATION_APP_ID = int(os.getenv("GITHUB_INTEGRATION_APP_ID") or 0) +GITHUB_INTEGRATION_APP_NAME = os.getenv("GITHUB_INTEGRATION_APP_NAME", "") GITHUB_INTEGRATION_APP_PRIVATE_KEY = read_secret("github_integration_app_private_key") GITHUB_INTEGRATION_APP_INSTALLATION_ID = int( os.getenv("GITHUB_INTEGRATION_APP_INSTALLATION_ID") or 0 ) -# client id and secret are only used for getting user access tokens to be able to push -# to the user's repositories. We are not re-using the regular oauth app in order to -# keep minimal permissions -GITHUB_INTEGRATION_APP_CLIENT_ID = os.getenv("GITHUB_INTEGRATION_APP_ID", "") -GITHUB_INTEGRATION_APP_CLIENT_SECRET = read_secret( - "github_integration_app_client_secret" -) GITHUB_MODEL_LIBRARY_ORG_NAME = os.getenv("GITHUB_MODEL_LIBRARY_ORG_NAME", "") GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT = os.getenv( "GITHUB_INDIVIDUAL_FILE_SIZE_LIMIT", 100 * 1024 * 1024 diff --git a/django/library/fs.py b/django/library/fs.py index 9eee5a731..0335e1c99 100644 --- a/django/library/fs.py +++ b/django/library/fs.py @@ -817,11 +817,7 @@ def __init__(self, codebase): self.mirror = codebase.git_mirror if not self.mirror: raise ValueError("Codebase must have a git_mirror") - self.repo_dir = Path(self.codebase.base_git_dir, str(self.repo_name)).absolute() - - @property - def repo_name(self): - return self.mirror.repository_name + self.repo_dir = Path(self.codebase.base_git_dir).absolute() @property def committer(self): @@ -1053,7 +1049,7 @@ def append_releases(self, releases=None) -> Repo: """ self.check_file_sizes(self.codebase) if not releases: - releases = self.mirror.unmirrored_local_releases + releases = self.mirror.unbuilt_releases if not releases: # nothing to do, return the existing repo return Repo(self.repo_dir) @@ -1061,7 +1057,7 @@ def append_releases(self, releases=None) -> Repo: # make sure the releases are higher than the latest mirrored release if not all( Version(release.version_number) - > Version(self.mirror.latest_local_release.version_number) + > Version(self.mirror.latest_built_release.version_number) for release in releases ): raise ValueError( @@ -1077,7 +1073,7 @@ def append_releases(self, releases=None) -> Repo: self.create_release_branch(release, commit) self.checkout_main() # record newly mirrored releases and update timestamp - self.mirror.update_local_releases(releases) + self.mirror.mark_releases_built(releases) return Repo(self.repo_dir) def build(self) -> Repo: @@ -1100,14 +1096,20 @@ def build(self) -> Repo: self.create_release_branch(release, commit) self.checkout_main() # record mirrored releases and update timestamp - self.mirror.update_local_releases(releases) + self.mirror.mark_releases_built(releases) return Repo(self.repo_dir) def update_or_build(self) -> Repo: - if self.repo_dir.exists() and self.repo_dir.joinpath(".git").exists(): - return self.append_releases() - else: + # if the repo doesn't exist, is empty, or the mirror object is not tracking them, + # build (or rebuild) the repo + if ( + not self.repo_dir.exists() + or not self.repo_dir.joinpath(".git").exists() + or not self.mirror.built_releases.exists() + ): return self.build() + else: + return self.append_releases() def dirs_equal(self, dir1: Path, dir2: Path, ignore=[".git"]): """ diff --git a/django/library/github_integration.py b/django/library/github_integration.py index 722d8a6b6..ffcc81ffb 100644 --- a/django/library/github_integration.py +++ b/django/library/github_integration.py @@ -2,12 +2,13 @@ from github import GithubIntegration, Auth, Github from github.GithubException import UnknownObjectException from github.Repository import Repository as GithubRepo +from github.AuthenticatedUser import AuthenticatedUser from git import Repo as GitRepo from django.conf import settings from django.core.cache import cache from django.utils import timezone -from .models import Codebase +from .models import Codebase, CodebaseGitRemote, GithubIntegrationAppInstallation INSTALLATION_ACCESS_TOKEN_REDIS_KEY = "github_installation_access_token" @@ -17,14 +18,11 @@ class GithubRepoNameValidator: def validate( cls, repo_name: str, - username: str | None = None, - user_access_token: str | None = None, + installation: GithubIntegrationAppInstallation | None = None, ): cls._validate_format(repo_name) - if username and user_access_token: - cls._check_user_repo_name_unused(repo_name, username, user_access_token) - elif username: - raise ValueError("User access token required for user repository") + if installation: + cls._check_user_repo_name_unused(repo_name, installation) else: cls._check_org_repo_name_unused(repo_name) @@ -43,15 +41,16 @@ def _validate_format(repo_name: str): @staticmethod def _check_user_repo_name_unused( - repo_name: str, username: str, user_access_token: str + repo_name: str, installation: GithubIntegrationAppInstallation ): + username = installation.github_login if username in repo_name: raise ValueError( f"Repository name cannot contain your username: '{username}'" ) - github = Github(user_access_token) + github = Github(GithubApi.get_user_installation_access_token(installation)) try: - github.get_user(username).get_repo(repo_name) + github.get_user().get_repo(repo_name) raise ValueError( f"Repository name already exists at https://github.com/{username}/{repo_name}" ) @@ -64,7 +63,7 @@ def _check_org_repo_name_unused(repo_name: str): raise ValueError( f"Repository name cannot contain the organization name: '{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}'" ) - github = Github(GithubApi.get_installation_access_token()) + github = Github(GithubApi.get_org_installation_access_token()) try: github.get_organization(settings.GITHUB_MODEL_LIBRARY_ORG_NAME).get_repo( repo_name @@ -84,28 +83,26 @@ class GithubApi: def __init__( self, codebase: Codebase, + remote: CodebaseGitRemote, local_repo: GitRepo, - repo_name: str, - is_user_repo=False, - organization_login: str | None = None, - user_access_token: str | None = None, - private_repo=False, ): - if is_user_repo: - raise NotImplementedError("User repositories not yet supported") - self.private_repo = private_repo self.codebase = codebase + self.remote = remote self.local_repo = local_repo - self.repo_name = repo_name - self.is_user_repo = is_user_repo - if is_user_repo and not organization_login: - raise ValueError("User access token required for user repository") - if not is_user_repo and not organization_login: - raise ValueError("Organization login required for org repository") - self.organization_login = organization_login - self.user_access_token = user_access_token self._github_repo = None + @property + def repo_owner(self): + return self.remote.owner + + @property + def repo_name(self): + return self.remote.repo_name + + @property + def is_user_repo(self): + return self.remote.is_user_repo + @property def github_repo(self) -> GithubRepo: if not self._github_repo: @@ -117,19 +114,35 @@ def github_repo(self) -> GithubRepo: @property def installation_access_token(self): - return self.get_installation_access_token() + if self.is_user_repo: + return self.get_user_installation_access_token(self.remote.installation) + return self.get_org_installation_access_token() + + @staticmethod + def get_user_installation_access_token( + installation: GithubIntegrationAppInstallation | None, + ) -> str | None: + if not installation: + return None + auth = Auth.AppAuth( + settings.GITHUB_INTEGRATION_APP_ID, + settings.GITHUB_INTEGRATION_APP_PRIVATE_KEY, + ) + integration = GithubIntegration(auth=auth) + installation_auth = integration.get_access_token(installation.installation_id) + return installation_auth.token @classmethod - def get_installation_access_token(cls): + def get_org_installation_access_token(cls) -> str: cached_token = cache.get(INSTALLATION_ACCESS_TOKEN_REDIS_KEY) if cached_token: return cached_token - return cls.refresh_installation_access_token() + return cls.refresh_org_installation_access_token() @staticmethod - def refresh_installation_access_token(): - """retrieve a new installation access token for the Github app - and cache it for future use + def refresh_org_installation_access_token() -> str: + """retrieve a new installation access token for the Github app installed + on the central CoMSES model library organization account and cache it for future use """ auth = Auth.AppAuth( settings.GITHUB_INTEGRATION_APP_ID, @@ -151,40 +164,22 @@ def refresh_installation_access_token(): ) return token - @staticmethod - def get_user_access_token(code: str): - # just need to link to the app install and it will go to callback with ?code=... - """return an access token for the Github user - - this token is used to authenticate requests to the Github API - to act on behalf of the user on resources they own - """ - github = Github() - app = github.get_oauth_application( - settings.GITHUB_INTEGRATION_APP_CLIENT_ID, - settings.GITHUB_INTEGRATION_APP_CLIENT_SECRET, - ) - return app.get_access_token(code).token - - def get_or_create_repo(self) -> GithubRepo: + def get_or_create_repo(self, private=False) -> GithubRepo: """get or create the Github repository for a user or organization""" try: return self.github_repo except: if self.is_user_repo: - self._github_repo = self._create_user_repo() + self._github_repo = self._create_user_repo(private) else: - self._github_repo = self._create_org_repo() + self._github_repo = self._create_org_repo(private) return self._github_repo - def push(self, local_repo: GitRepo): + def push(self, local_repo: GitRepo) -> str: """push the local git repository to the Github repository""" - if self.is_user_repo: - raise NotImplementedError("User repositories not yet supported") - else: - token = self.installation_access_token - push_url = f"https://x-access-token:{token}@github.com/{self.github_repo.full_name}.git" - self._push_to_url(local_repo, push_url) + token = self.installation_access_token + push_url = f"https://x-access-token:{token}@github.com/{self.github_repo.full_name}.git" + return self._push_to_url(local_repo, push_url) def create_releases(self, local_repo: GitRepo): """create Github releases for each tag in the local repository that @@ -205,31 +200,24 @@ def create_releases(self, local_repo: GitRepo): def _get_existing_repo(self): """attempt to get an existing repository for the authenticated user or organization""" - if self.is_user_repo: - github = Github(self.user_access_token) - name = github.get_user().login - return github.get_repo(f"{name}/{self.repo_name}") - else: - github = Github(self.installation_access_token) - return github.get_repo(f"{self.organization_login}/{self.repo_name}") + github = Github(self.installation_access_token) + return github.get_repo(f"{self.repo_owner}/{self.repo_name}") - def _create_user_repo(self): + def _create_user_repo(self, private=False): """create a new repository in the user's account - this function requires the `repo` scope for the user access token + this function requires the `repo` scope for the installation access token """ - token = self.user_access_token - if not token: - raise ValueError("User access token required for creating user repository") + token = self.installation_access_token github = Github(token) repo = github.get_user().create_repo( name=self.repo_name, - description=self.codebase.description, - private=self.private_repo, + description=f"Mirror of {self.codebase.permanent_url}", + private=private, ) return repo - def _create_org_repo(self): + def _create_org_repo(self, private=False): """create a new repository in the CoMSES model library organization this function requires the `repo` scope for the installation access token @@ -240,14 +228,24 @@ def _create_org_repo(self): repo = org.create_repo( name=self.repo_name, description=f"Mirror of {self.codebase.permanent_url}", - private=self.private_repo, + private=private, ) return repo - def _push_to_url(self, local_repo: GitRepo, push_url: str): + def _push_to_url(self, local_repo: GitRepo, push_url: str) -> str: if "origin" not in local_repo.remotes: local_repo.create_remote("origin", push_url) else: local_repo.remotes["origin"].set_url(push_url) - local_repo.git.push("--all") - local_repo.git.push("--tags") + # https://gitpython.readthedocs.io/en/stable/reference.html#git.remote.PushInfo + result_all = local_repo.git.push(["--all"]) + result_tags = local_repo.git.push(["--tags"]) + summaries = [] + for result in (result_all, result_tags): + if result: # result will be None if the push failed entirely + for info in result: + if info.summary: + summaries.append(info.summary) + if not summaries: + return "push failed entirely" + return "\n".join(summaries) diff --git a/django/library/metadata.py b/django/library/metadata.py index 40b382708..7b4dc19ea 100644 --- a/django/library/metadata.py +++ b/django/library/metadata.py @@ -141,8 +141,9 @@ def _common_codebase_fields(cls, codebase) -> dict: type_="SoftwareSourceCode", name=codebase.title, codeRepository=( - codebase.git_mirror.remote_url - if codebase.git_mirror + codebase.git_mirror.remotes.first().url + # FIXME: coderepository can be one url, how do we determine priority? + if codebase.git_mirror and codebase.git_mirror.remotes.first() else codebase.repository_url ) or None, diff --git a/django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py b/django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py deleted file mode 100644 index 3d8510af1..000000000 --- a/django/library/migrations/0033_codebasegitmirror_codebase_git_mirror.py +++ /dev/null @@ -1,69 +0,0 @@ -# Generated by Django 4.2.17 on 2025-01-09 20:57 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("library", "0032_license_text_codemeta_snapshot"), - ] - - operations = [ - migrations.CreateModel( - name="CodebaseGitMirror", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("date_created", models.DateTimeField(auto_now_add=True)), - ("last_modified", models.DateTimeField(auto_now=True)), - ("repository_name", models.CharField(max_length=100, unique=True)), - ( - "remote_url", - models.URLField( - blank=True, help_text="URL of mirrored remote repository" - ), - ), - ("last_local_update", models.DateTimeField(blank=True, null=True)), - ("last_remote_update", models.DateTimeField(blank=True, null=True)), - ( - "user_access_token", - models.CharField(blank=True, max_length=200, null=True), - ), - ( - "organization_login", - models.CharField(blank=True, max_length=100, null=True), - ), - ( - "local_releases", - models.ManyToManyField( - related_name="+", to="library.codebaserelease" - ), - ), - ( - "remote_releases", - models.ManyToManyField( - related_name="+", to="library.codebaserelease" - ), - ), - ], - ), - migrations.AddField( - model_name="codebase", - name="git_mirror", - field=models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="codebase", - to="library.codebasegitmirror", - ), - ), - ] diff --git a/django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py b/django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py new file mode 100644 index 000000000..a7f8db4ec --- /dev/null +++ b/django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py @@ -0,0 +1,163 @@ +# Generated by Django 4.2.17 on 2025-01-25 01:34 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("library", "0032_license_text_codemeta_snapshot"), + ] + + operations = [ + migrations.CreateModel( + name="CodebaseGitMirror", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("last_modified", models.DateTimeField(auto_now=True)), + ( + "built_releases", + models.ManyToManyField( + related_name="+", to="library.codebaserelease" + ), + ), + ], + ), + migrations.CreateModel( + name="GithubIntegrationAppInstallation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("github_user_id", models.BigIntegerField(unique=True)), + ("github_login", models.CharField(max_length=255)), + ("installation_id", models.BigIntegerField(unique=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("last_modified", models.DateTimeField(auto_now=True)), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="github_integration_app_installation", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.CreateModel( + name="CodebaseGitRemote", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "should_push", + models.BooleanField( + default=True, help_text="Whether to push to this remote" + ), + ), + ( + "is_user_repo", + models.BooleanField( + default=False, help_text="Whether this is a user-owned remote" + ), + ), + ( + "should_archive", + models.BooleanField( + default=False, + help_text="Whether to archive new releases from this remote", + ), + ), + ("repo_name", models.CharField(max_length=100)), + ( + "owner", + models.CharField( + blank=True, + help_text="Github username (or organization) who owns the remote repository", + max_length=100, + null=True, + ), + ), + ( + "url", + models.URLField( + blank=True, help_text="http URL of remote repository" + ), + ), + ("last_push_log", models.TextField(blank=True)), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("last_modified", models.DateTimeField(auto_now=True)), + ( + "mirror", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="remotes", + to="library.codebasegitmirror", + ), + ), + ], + ), + migrations.AddField( + model_name="codebase", + name="git_mirror", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="codebase", + to="library.codebasegitmirror", + ), + ), + migrations.AddConstraint( + model_name="codebasegitremote", + constraint=models.UniqueConstraint( + condition=models.Q(("is_user_repo", True), ("should_push", True)), + fields=("mirror",), + name="single_pushable_user_repo", + ), + ), + migrations.AddConstraint( + model_name="codebasegitremote", + constraint=models.UniqueConstraint( + condition=models.Q(("is_user_repo", False), ("should_push", True)), + fields=("mirror",), + name="single_pushable_org_repo", + ), + ), + migrations.AddConstraint( + model_name="codebasegitremote", + constraint=models.UniqueConstraint( + condition=models.Q(("should_archive", True)), + fields=("mirror",), + name="single_archivable_repo", + ), + ), + migrations.AlterUniqueTogether( + name="codebasegitremote", + unique_together={("mirror", "owner")}, + ), + ] diff --git a/django/library/models.py b/django/library/models.py index 51da5fbe3..83aa03583 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -13,11 +13,12 @@ from collections import defaultdict from datetime import date, timedelta +from allauth.socialaccount.models import SocialAccount from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.core.files.images import ImageFile from django.core.files.storage import FileSystemStorage -from django.db import models, transaction +from django.db import IntegrityError, models, transaction from django.db.models import Prefetch, Q, Count, Max from django.utils.functional import cached_property from django.urls import reverse @@ -616,62 +617,148 @@ def updated_after(self, start_date, end_date=None, **kwargs): return new_codebases, updated_codebases, releases -class CodebaseGitMirror(models.Model): - """ - Keeps track of a git repository and its GitHub remote that were created - from a Codebase using the mirror (read-only archiving) workflow - """ +class GithubIntegrationAppInstallation(models.Model): + """Represents a user's installation of the Github Integration App""" - # is_active = models.BooleanField(default=True) + user = models.OneToOneField( + settings.AUTH_USER_MODEL, + related_name="github_integration_app_installation", + on_delete=models.CASCADE, + ) + github_user_id = models.BigIntegerField(unique=True) + github_login = models.CharField(max_length=255) + installation_id = models.BigIntegerField(unique=True) date_created = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) - repository_name = models.CharField(max_length=100, unique=True) - remote_url = models.URLField( + + +class CodebaseGitRemote(models.Model): + """Represents a remote repository that a local git mirror can push to""" + + should_push = models.BooleanField( + default=True, help_text="Whether to push to this remote" + ) + is_user_repo = models.BooleanField( + default=False, help_text="Whether this is a user-owned remote" + ) + should_archive = models.BooleanField( + default=False, help_text="Whether to archive new releases from this remote" + ) + repo_name = models.CharField(max_length=100) + owner = models.CharField( + max_length=100, + null=True, + blank=True, + help_text="Github username (or organization) who owns the remote repository", + ) + url = models.URLField( blank=True, - help_text=_("URL of mirrored remote repository"), + help_text=_("http URL of remote repository"), ) - # keep track of timestamp and releases that have been mirrored locally - last_local_update = models.DateTimeField(null=True, blank=True) - local_releases = models.ManyToManyField("CodebaseRelease", related_name="+") - # keep track of timestamp and releases that have been synced to the remote - last_remote_update = models.DateTimeField(null=True, blank=True) - remote_releases = models.ManyToManyField("CodebaseRelease", related_name="+") - user_access_token = models.CharField(max_length=200, null=True, blank=True) - organization_login = models.CharField(max_length=100, null=True, blank=True) + mirror = models.ForeignKey( + "CodebaseGitMirror", + related_name="remotes", + on_delete=models.CASCADE, + ) + last_push_log = models.TextField(blank=True) + date_created = models.DateTimeField(auto_now_add=True) + last_modified = models.DateTimeField(auto_now=True) @property - def latest_local_release(self): - return max(self.local_releases.all(), key=lambda r: Version(r.version_number)) + def installation(self) -> GithubIntegrationAppInstallation | None: + if self.is_user_repo: + return self.mirror.codebase.submitter.github_integration_app_installation + return None - @property - def latest_remote_release(self): - return max(self.remote_releases.all(), key=lambda r: Version(r.version_number)) + def __str__(self): + return f"{self.owner}/{self.repo_name} (active={self.should_push}) (sync={self.should_archive})" + + class Meta: + unique_together = [ + ("mirror", "owner"), + ("owner", "repo_name"), + ] + unique_together = ("mirror", "owner") + constraints = [ + models.UniqueConstraint( + fields=["mirror"], + condition=models.Q(should_push=True, is_user_repo=True), + name="single_pushable_user_repo", + ), + models.UniqueConstraint( + fields=["mirror"], + condition=models.Q(should_push=True, is_user_repo=False), + name="single_pushable_org_repo", + ), + models.UniqueConstraint( + fields=["mirror"], + condition=models.Q(should_archive=True), + name="single_archivable_repo", + ), + ] + + +class CodebaseGitMirror(models.Model): + """Represents a local git repository that mirrors a codebase's + regular (as opposed to pulled from github) published releases. It can have + two remotes: one on the submitter's account, and one in the CoMSES model library + organization + """ + + built_releases = models.ManyToManyField("CodebaseRelease", related_name="+") + date_created = models.DateTimeField(auto_now_add=True) + last_modified = models.DateTimeField(auto_now=True) @property - def unmirrored_local_releases(self): - return self.codebase.public_releases().exclude( - id__in=self.local_releases.values_list("id", flat=True) - ) + def latest_built_release(self): + if not self.built_releases.all(): + return None + return max(self.built_releases.all(), key=lambda r: Version(r.version_number)) @property - def unmirrored_remote_releases(self): - return self.local_releases.exclude( - id__in=self.remote_releases.values_list("id", flat=True) + def unbuilt_releases(self): + return self.codebase.public_releases().exclude( + id__in=self.built_releases.values_list("id", flat=True) ) - def update_local_releases(self, new_releases: models.QuerySet | list): - if self.local_releases.exists(): - self.local_releases.add(*new_releases) - else: - self.local_releases.set(new_releases) - self.last_local_update = timezone.now() + def mark_releases_built(self, new_releases: models.QuerySet | list): + self.built_releases.add(*new_releases) self.save() - def update_remote_releases(self): - releases = self.local_releases.all() - self.remote_releases.set(releases) - self.last_remote_update = timezone.now() - self.save() + def create_remote(self, owner, repo_name, **kwargs): + """create a new remote for this mirror, or get an existing one if there is a + matching remote that failed to actually be created at the remote location""" + try: + existing_remote = CodebaseGitRemote.objects.filter( + mirror=self, + owner=owner, + repo_name=repo_name, + url__isnull=False, # proxy for a successful remote creation + ).first() + if existing_remote: + return existing_remote + remote = CodebaseGitRemote.objects.create( + mirror=self, + owner=owner, + repo_name=repo_name, + **kwargs, + ) + return remote + except IntegrityError as e: + if "single_pushable_user_repo" in str(e): + raise ValidationError( + "There can only be one active user-owned repository." + ) + if "single_pushable_org_repo" in str(e): + raise ValidationError( + "There can only be one active organization-owned repository." + ) + if "single_archivable_repo" in str(e): + raise ValidationError( + "There can only be one repository that releases are being archived from." + ) + else: + raise ValidationError("A repository already exists at this location.") @add_to_comses_permission_whitelist @@ -898,14 +985,13 @@ def base_library_dir(self): def base_git_dir(self): return pathlib.Path(settings.REPOSITORY_ROOT, str(self.uuid)) - def create_git_mirror(self, repository_name, **kwargs): + def create_git_mirror(self, **kwargs): if not self.git_mirror: git_mirror = CodebaseGitMirror.objects.create( - repository_name=repository_name, **kwargs, ) self.git_mirror = git_mirror - self.save() + self.save(rebuild_metadata=False) return self.git_mirror @property @@ -1987,9 +2073,11 @@ def publish(self): self.id, dry_run=settings.DEPLOY_ENVIRONMENT.is_development ) if self.codebase.git_mirror: - from .tasks import update_mirrored_codebase + from .tasks import add_releases_and_push_codebase_repo - transaction.on_commit(lambda: update_mirrored_codebase(self.codebase.id)) + transaction.on_commit( + lambda: add_releases_and_push_codebase_repo(self.codebase.id) + ) def _publish(self): if not self.live: diff --git a/django/library/tasks.py b/django/library/tasks.py index 31e7073d5..b48fdc024 100644 --- a/django/library/tasks.py +++ b/django/library/tasks.py @@ -11,61 +11,53 @@ @db_task(retries=3, retry_delay=30) -def mirror_codebase(codebase_id: int, private_repo=False): - """asynchronous task that mirrors a codebase to a remote Github repository""" +def build_and_push_codebase_repo(codebase_id: int, private_repo=False): codebase = Codebase.objects.get(id=codebase_id) mirror = codebase.git_mirror if not mirror: raise ValueError("Codebase does not have a git mirror") - mirror.organization_login = settings.GITHUB_MODEL_LIBRARY_ORG_NAME - mirror.save() git_fs_api = CodebaseGitRepositoryApi(codebase) local_repo = git_fs_api.update_or_build() - gh_api = GithubApi( - codebase=codebase, - local_repo=local_repo, - repo_name=mirror.repository_name, - is_user_repo=False, - organization_login=mirror.organization_login, - user_access_token=mirror.user_access_token, - private_repo=private_repo, - ) - repo = gh_api.get_or_create_repo() - mirror.remote_url = repo.html_url - gh_api.push(local_repo) - gh_api.create_releases(local_repo) - mirror.update_remote_releases() + for remote in mirror.remotes.filter(should_push=True): + gh_api = GithubApi( + codebase=codebase, + local_repo=local_repo, + remote=remote, + ) + repo = gh_api.get_or_create_repo(private=private_repo) + remote.url = repo.html_url + push_summary = gh_api.push(local_repo) + remote.last_push_log = push_summary + remote.save() + gh_api.create_releases(local_repo) @db_task(retries=3, retry_delay=30) -def update_mirrored_codebase(codebase_id: int): - """asynchronous task that updates a mirrored codebase by pushing new releases to Github""" +def add_releases_and_push_codebase_repo(codebase_id: int): codebase = Codebase.objects.get(id=codebase_id) mirror = codebase.git_mirror if not mirror: raise ValueError("Codebase does not have a git mirror") - if not mirror.remote_url: - raise ValueError("Codebase git mirror does not have a remote url") git_fs_api = CodebaseGitRepositoryApi(codebase) local_repo = git_fs_api.append_releases() - gh_api = GithubApi( - codebase=codebase, - local_repo=local_repo, - repo_name=mirror.repository_name, - is_user_repo=bool(mirror.user_access_token), - organization_login=mirror.organization_login, - user_access_token=mirror.user_access_token, - ) - gh_api.push(local_repo) - gh_api.create_releases(local_repo) - mirror.update_remote_releases() + + for remote in mirror.remotes.filter(should_push=True): + gh_api = GithubApi( + codebase=codebase, + local_repo=local_repo, + remote=remote, + ) + push_summary = gh_api.push(local_repo) + remote.last_push_log = push_summary + remote.save() + gh_api.create_releases(local_repo) @db_task(retries=3, retry_delay=30) -def update_mirrored_release_metadata(release_id: int): +def update_release_branch_and_push_codebase_repo(release_id: int): """asynchronous task that updates a SINGLE RELEASE BRANCH with any metadata changes that may have occurred. @@ -76,22 +68,21 @@ def update_mirrored_release_metadata(release_id: int): mirror = codebase.git_mirror if not mirror: raise ValueError("Codebase does not have a git mirror") - if not mirror.remote_url: - raise ValueError("Codebase git mirror does not have a remote url") git_fs_api = CodebaseGitRepositoryApi(codebase) local_repo = git_fs_api.update_release_branch(release) + + # local_repo will be None if no changes were made if local_repo: - gh_api = GithubApi( - codebase=codebase, - local_repo=local_repo, - repo_name=mirror.repository_name, - is_user_repo=bool(mirror.user_access_token), - organization_login=mirror.organization_login, - user_access_token=mirror.user_access_token, - ) - gh_api.push(local_repo) - mirror.update_remote_releases() + for remote in mirror.remotes.filter(should_push=True): + gh_api = GithubApi( + codebase=codebase, + local_repo=local_repo, + remote=remote, + ) + push_summary = gh_api.push(local_repo) + remote.last_push_log = push_summary + remote.save() @db_task(retries=1, retry_delay=30) @@ -104,8 +95,8 @@ def update_fs_release_metadata(release_id: int): fs_api.rebuild(metadata_only=True) # if the release is published and the codebase has a git mirror, # update the metadata in the git repository - if release.is_published and codebase.git_mirror and codebase.git_mirror.remote_url: - update_mirrored_release_metadata(release_id) + if release.is_published and codebase.git_mirror: + update_release_branch_and_push_codebase_repo(release_id) @on_commit_task() diff --git a/django/library/tests/test_fs.py b/django/library/tests/test_fs.py index d7923db8e..078cc281f 100644 --- a/django/library/tests/test_fs.py +++ b/django/library/tests/test_fs.py @@ -102,7 +102,7 @@ def setUp(self): self.codebase_factory = CodebaseFactory(submitter=self.submitter) self.codebase = self.codebase_factory.create() self.release_1 = self.codebase.create_release() - self.git_mirror = self.codebase.create_git_mirror("animals-model") + self.git_mirror = self.codebase.create_git_mirror() def tearDown(self): clear_test_shared_folder(settings.REPOSITORY_ROOT) From a3738fdd40fd23799e19ba99af500eeffd0165a3 Mon Sep 17 00:00:00 2001 From: sgfost Date: Mon, 27 Jan 2025 20:12:56 -0700 Subject: [PATCH 06/66] feat: views/routing for github integration config --- .../jinja2/library/codebases/git.jinja | 19 ++ .../library/codebases/releases/retrieve.jinja | 6 +- ..._alter_codebasegitremote_owner_and_more.py | 30 +++ django/library/models.py | 10 +- django/library/serializers.py | 24 +++ django/library/urls.py | 5 + django/library/views.py | 175 ++++++++++++++---- frontend/src/composables/api/git.ts | 51 +++++ frontend/src/types.ts | 28 +++ 9 files changed, 305 insertions(+), 43 deletions(-) create mode 100644 django/library/jinja2/library/codebases/git.jinja create mode 100644 django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py create mode 100644 frontend/src/composables/api/git.ts diff --git a/django/library/jinja2/library/codebases/git.jinja b/django/library/jinja2/library/codebases/git.jinja new file mode 100644 index 000000000..ea41b47fd --- /dev/null +++ b/django/library/jinja2/library/codebases/git.jinja @@ -0,0 +1,19 @@ +{% extends "base.jinja" %} +{% from "common.jinja" import breadcrumb %} + +{% block title %}GitHub Integration{% endblock %} + +{% block introduction %}

GitHub Integration Configuration

{% endblock %} + +{% block content %} +{{ breadcrumb([ +{'url': url('library:codebase-list'), 'text': 'Computational Model Library' }, +{'url': url('library:codebase-detail', codebase.identifier), 'text': codebase.title|truncate(68) }, +{'text': 'GitHub Integration' }, +]) }} +
+{% endblock %} + +{% block js %} +{{ vite_asset("apps/github_configuration.ts") }} +{% endblock %} \ No newline at end of file diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index d50e518ea..7f635bc96 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -301,12 +301,14 @@ beta
- {% if codebase.git_mirror %} + {% if False %} + {# FIXME: multiple remotes! show all of them #} + {# and link to github configuration page #} {% if codebase.git_mirror.remote_url %}
- {{ codebase.git_mirror.repository_name }} + {{ codebase.git_mirror.repo_name }} diff --git a/django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py b/django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py new file mode 100644 index 000000000..169339c2b --- /dev/null +++ b/django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.17 on 2025-01-28 02:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("library", "0033_codebasegitmirror_githubintegrationappinstallation_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="codebasegitremote", + name="owner", + field=models.CharField( + help_text="Github username (or organization) who owns the remote repository", + max_length=100, + ), + ), + migrations.AddConstraint( + model_name="codebasegitremote", + constraint=models.CheckConstraint( + check=models.Q( + ("is_user_repo", True), ("should_archive", False), _connector="OR" + ), + name="org_repo_not_archivable", + ), + ), + ] diff --git a/django/library/models.py b/django/library/models.py index 83aa03583..3c15f9b2a 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -647,8 +647,6 @@ class CodebaseGitRemote(models.Model): repo_name = models.CharField(max_length=100) owner = models.CharField( max_length=100, - null=True, - blank=True, help_text="Github username (or organization) who owns the remote repository", ) url = models.URLField( @@ -695,6 +693,10 @@ class Meta: condition=models.Q(should_archive=True), name="single_archivable_repo", ), + models.CheckConstraint( + check=models.Q(is_user_repo=True) | models.Q(should_archive=False), + name="org_repo_not_archivable", + ) ] @@ -757,6 +759,10 @@ def create_remote(self, owner, repo_name, **kwargs): raise ValidationError( "There can only be one repository that releases are being archived from." ) + if "org_repo_not_archivable" in str(e): + raise ValidationError( + "Repositories in the CoMSES Model Library organization cannot be archived." + ) else: raise ValidationError("A repository already exists at this location.") diff --git a/django/library/serializers.py b/django/library/serializers.py index 43b493f5e..208d46504 100644 --- a/django/library/serializers.py +++ b/django/library/serializers.py @@ -30,6 +30,7 @@ RelatedUserSerializer, ) from .models import ( + CodebaseGitRemote, PeerReviewer, ReleaseContributor, Codebase, @@ -504,6 +505,29 @@ class Meta: ) +class CodebaseGitRemoteSerializer(serializers.ModelSerializer): + class Meta: + model = CodebaseGitRemote + fields = ( + "id", + "owner", + "repo_name", + "url", + "should_push", + "should_archive", + "is_user_repo", + "last_push_log", + ) + read_only_fields = ( + "id", + "owner", + "repo_name", + "url", + "is_user_repo", + "last_push_log", + ) + + class CodebaseImageSerializer(serializers.ModelSerializer): identifier = serializers.IntegerField(source="id") name = serializers.CharField(source="title") diff --git a/django/library/urls.py b/django/library/urls.py index dc6a838a9..b19556e80 100644 --- a/django/library/urls.py +++ b/django/library/urls.py @@ -14,6 +14,11 @@ router.register( r"codebases/(?P[\w\-.]+)/releases", views.CodebaseReleaseViewSet ) +router.register( + r"codebases/(?P[\w\-.]+)/git/remotes", + views.CodebaseGitRemoteViewSet, + basename="codebase-git-remotes", +) router.register(r"reviewers", views.PeerReviewerViewSet), router.register(r"programming-languages", views.ProgrammingLanguageViewSet), router.register( diff --git a/django/library/views.py b/django/library/views.py index da576312b..52b04f265 100644 --- a/django/library/views.py +++ b/django/library/views.py @@ -1,4 +1,5 @@ import json +from allauth.socialaccount.models import SocialAccount from django.forms import Form from django.conf import settings from django.contrib import messages @@ -28,6 +29,7 @@ from rest_framework.exceptions import ( PermissionDenied as DrfPermissionDenied, ValidationError, + NotFound, ) from rest_framework.filters import OrderingFilter from rest_framework.permissions import IsAuthenticatedOrReadOnly @@ -56,13 +58,13 @@ PeerReviewFilterForm, ) from .fs import ( - CodebaseGitRepositoryApi, FileCategoryDirectories, StagingDirectories, MessageLevels, ) from .models import ( Codebase, + CodebaseGitRemote, CodebaseRelease, Contributor, CodebaseImage, @@ -76,6 +78,7 @@ ) from .permissions import CodebaseReleaseUnpublishedFilePermissions from .serializers import ( + CodebaseGitRemoteSerializer, CodebaseSerializer, CodebaseReleaseSerializer, ContributorSerializer, @@ -89,7 +92,7 @@ PeerReviewFeedbackEditorSerializer, PeerReviewEventLogSerializer, ) -from .tasks import mirror_codebase, update_mirrored_codebase +from .tasks import build_and_push_codebase_repo import logging import pathlib @@ -528,41 +531,6 @@ def retrieve(self, request, *args, **kwargs): data = add_user_retrieve_perms(instance, serializer.data, request.user) return Response(data) - # @action(detail=True, methods=["post"]) - # def github_sync(self, request, *args, **kwargs): - # pass - - @action(detail=True, methods=["post"]) - def github_mirror(self, request, *args, **kwargs): - codebase = self.get_object() - if not codebase.live: - raise ValidationError("This model does not have any published releases") - if codebase.git_mirror: - raise ValidationError("This codebase is already mirrored to a GitHub repo") - CodebaseGitRepositoryApi.check_file_sizes(codebase) - repo_name = request.data.get("repo_name") - try: - GithubRepoNameValidator.validate(repo_name) - except ValueError as e: - raise ValidationError(str(e)) - codebase.create_git_mirror(repo_name) - mirror_codebase(codebase.id, private_repo=settings.DEBUG) - return Response( - status=status.HTTP_202_ACCEPTED, - data="Mirroring process started, this should take only a few seconds. Refresh this page to see the status.", - ) - - @action(detail=True, methods=["post"]) - def update_github_mirror(self, request, *args, **kwargs): - codebase = self.get_object() - if not codebase.git_mirror: - return Response( - data={"error": "This codebase is not mirrored to a GitHub repo"}, - status=status.HTTP_400_BAD_REQUEST, - ) - update_mirrored_codebase(codebase.id) - return Response(status=status.HTTP_202_ACCEPTED) - class DevelopmentCodebaseDeleteView(mixins.DestroyModelMixin, CodebaseViewSet): lookup_field = "identifier" @@ -611,18 +579,147 @@ def get_redirect_url(self, *args, **kwargs): return super().get_redirect_url(*args, **kwargs) -class CodebaseImagePermission(permissions.BasePermission): +class CanChangeCodebase(permissions.BasePermission): def has_permission(self, request, view): codebase = get_object_or_404(Codebase, identifier=view.kwargs["identifier"]) return request.user.has_perm("library.change_codebase", codebase) +class CodebaseGitRemoteViewSet( + mixins.RetrieveModelMixin, mixins.UpdateModelMixin, viewsets.GenericViewSet +): + queryset = CodebaseGitRemote.objects.all() + permission_classes = (CanChangeCodebase,) + pagination_class = None + serializer_class = CodebaseGitRemoteSerializer + + def get_queryset(self): + resolved = resolve(self.request.path) + identifier = resolved.kwargs["identifier"] + return self.queryset.filter(mirror__codebase__identifier=identifier) + + def get_codebase(self): + try: + return Codebase.objects.get(identifier=self.kwargs.get("identifier")) + except Codebase.DoesNotExist: + raise NotFound("Codebase not found.") + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset() + if request.accepted_renderer.format == "html": + context = {"codebase": self.get_codebase(), "remotes": queryset} + return Response( + context, + template_name="library/codebases/git.jinja", + ) + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + @action(detail=False, methods=["get"]) + def submitter_installation_status(self, request, *args, **kwargs): + codebase = self.get_codebase() + submitter = codebase.submitter + installation_url = None + try: + social_account = submitter.member_profile.get_social_account("github") + github_account = { + "id": social_account.uid, + "username": social_account.extra_data.get("login"), + "profile_url": social_account.get_profile_url(), + } + except SocialAccount.DoesNotExist: + github_account = None + + if github_account: + if submitter.github_integration_app_installation: + github_account["installation_id"] = ( + submitter.github_integration_app_installation.installation_id + ) + else: + installation_url = f"https://github.com/apps/{settings.GITHUB_INTEGRATION_APP_NAME}/installations/new?target_id={github_account['id']}" + + return Response( + data={ + "github_account": github_account, + "installation_url": installation_url, + }, + status=status.HTTP_200_OK, + ) + + @action(detail=False, methods=["post"]) + def setup_user_github_remote(self, request, *args, **kwargs): + codebase = self.get_codebase() + installation = codebase.submitter.github_integration_app_installation + return self._setup_github_remote( + codebase=codebase, + repo_name=request.data.get("repo_name"), + owner=installation.github_login, + is_user_repo=True, + should_push=True, + should_archive=True, + installation=installation, + ) + + @action(detail=False, methods=["post"]) + def setup_org_github_remote(self, request, *args, **kwargs): + codebase = self.get_codebase() + return self._setup_github_remote( + codebase=codebase, + repo_name=request.data.get("repo_name"), + owner=settings.GITHUB_MODEL_LIBRARY_ORG_NAME, + is_user_repo=False, + should_push=True, + should_archive=False, + ) + + def _setup_github_remote( + self, + codebase: Codebase, + repo_name: str, + owner: str, + is_user_repo: bool, + should_push: bool, + should_archive: bool, + installation=None, + ): + if is_user_repo and not installation: + raise ValidationError( + "Installation of the Github integration app is required" + ) + if not codebase.live: + raise ValidationError("This model does not have any published releases") + if not repo_name: + raise ValidationError("Repository name is required") + try: + GithubRepoNameValidator.validate(repo_name, installation=installation) + except ValueError as e: + raise ValidationError(str(e)) + + # create a mirror if it doesn't exist + if not codebase.git_mirror: + codebase.create_git_mirror() + # set up the org remote + codebase.git_mirror.create_remote( + owner=owner, + repo_name=repo_name, + should_push=should_push, + is_user_repo=is_user_repo, + should_archive=should_archive, + ) + # trigger the build and push task + build_and_push_codebase_repo(codebase.id, private_repo=settings.DEBUG) + return Response( + status=status.HTTP_202_ACCEPTED, + data="Synchronization process started, this should take only a few seconds. Refresh this page to see the status.", + ) + + class CodebaseImageViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): lookup_field = "codebaseimage_id" lookup_value_regex = r"\d+" queryset = CodebaseImage.objects.all() serializer_class = CodebaseImageSerializer - permission_classes = (CodebaseImagePermission,) + permission_classes = (CanChangeCodebase,) def get_queryset(self): resolved = resolve(self.request.path) diff --git a/frontend/src/composables/api/git.ts b/frontend/src/composables/api/git.ts new file mode 100644 index 000000000..45bf7eaba --- /dev/null +++ b/frontend/src/composables/api/git.ts @@ -0,0 +1,51 @@ +import { toRefs } from "vue"; +import { useAxios, joinPaths, type RequestOptions } from "@/composables/api"; +import type { CodebaseGitRemote, CodebaseGitRemoteForm, GitIntegrationStatus } from "@/types"; +import type { AxiosResponse } from "axios"; + +export function useGitRemotesAPI(codebaseIdentifier: string) { + /** + * Composable function for making requests to the codebase release API + * + * @returns - An object containing reactive state of the request and helper functions for API requests + */ + + const baseUrl = `/codebases/${codebaseIdentifier}/git/remotes/`; + const { state, get, post, put, detailUrl } = useAxios(baseUrl); + + function url(paths: string[] = []) { + return joinPaths([baseUrl, ...paths]); + } + + async function submitterInstallationStatus( + options?: RequestOptions + ): Promise> { + return get(url(["submitter_installation_status"]), options); + } + + async function list(options?: RequestOptions): Promise> { + return get(baseUrl, options); + } + + async function update(id: number, data: CodebaseGitRemoteForm, options?: RequestOptions) { + return put(detailUrl(id), data, options); + } + + async function setupOrgGithubRemote(repoName: string, options?: RequestOptions) { + return post(url(["setup_org_github_remote"]), { repoName }, options); + } + + async function setupUserGithubRemote(repoName: string, options?: RequestOptions) { + return post(url(["setup_user_github_remote"]), { repoName }, options); + } + + return { + ...toRefs(state), + detailUrl, + list, + update, + submitterInstallationStatus, + setupOrgGithubRemote, + setupUserGithubRemote, + }; +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 8c9e1139d..e93db1a53 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -402,3 +402,31 @@ export interface FeedProps { datePrefix?: string; authorPrefix?: string; } + +export interface GithubAccount { + id: number; + username: string; + profileUrl: string; + installationId?: number; +} + +export interface GitIntegrationStatus { + githubAccount: GithubAccount | null; + installationUrl: string | null; +} + +export interface CodebaseGitRemote { + id: number; + owner: string; + repoName: string; + url: string; + shouldPush: boolean; + shouldArchive: boolean; + isUserRepo: boolean; + lastPushLog: string; +} + +export interface CodebaseGitRemoteForm { + shouldPush?: boolean; + shouldArchive?: boolean; +} From bca3816c9323b6faf1bdf04938d0d34cc91e6d6b Mon Sep 17 00:00:00 2001 From: sgfost Date: Tue, 28 Jan 2025 17:39:34 -0700 Subject: [PATCH 07/66] feat: add github sync configuration page this replaces the simple modal form to give better control over the feature --- .../jinja2/library/codebases/git.jinja | 20 ++- .../library/codebases/releases/retrieve.jinja | 88 +++++------ django/library/models.py | 14 +- django/library/serializers.py | 4 + django/library/views.py | 13 +- frontend/src/apps/github_mirror.ts | 8 - frontend/src/apps/github_sync.ts | 8 + frontend/src/components/GitHubRemoteItem.vue | 13 ++ .../components/GitHubSyncConfiguration.vue | 147 ++++++++++++++++++ frontend/src/components/GithubMirrorModal.vue | 93 ----------- frontend/src/composables/api/codebase.ts | 5 - frontend/src/composables/api/git.ts | 12 +- frontend/src/types.ts | 4 +- 13 files changed, 260 insertions(+), 169 deletions(-) delete mode 100644 frontend/src/apps/github_mirror.ts create mode 100644 frontend/src/apps/github_sync.ts create mode 100644 frontend/src/components/GitHubRemoteItem.vue create mode 100644 frontend/src/components/GitHubSyncConfiguration.vue delete mode 100644 frontend/src/components/GithubMirrorModal.vue diff --git a/django/library/jinja2/library/codebases/git.jinja b/django/library/jinja2/library/codebases/git.jinja index ea41b47fd..6036ed4f7 100644 --- a/django/library/jinja2/library/codebases/git.jinja +++ b/django/library/jinja2/library/codebases/git.jinja @@ -1,19 +1,27 @@ {% extends "base.jinja" %} {% from "common.jinja" import breadcrumb %} -{% block title %}GitHub Integration{% endblock %} +{% block title %}GitHub Sync{% endblock %} -{% block introduction %}

GitHub Integration Configuration

{% endblock %} +{% block introduction %}

GitHub Sync

{% endblock %} {% block content %} {{ breadcrumb([ {'url': url('library:codebase-list'), 'text': 'Computational Model Library' }, {'url': url('library:codebase-detail', codebase.identifier), 'text': codebase.title|truncate(68) }, -{'text': 'GitHub Integration' }, +{'text': 'GitHub Sync' }, ]) }} -
+

+ Configure GitHub Sync for {{ codebase.title }} +

+
{% endblock %} {% block js %} -{{ vite_asset("apps/github_configuration.ts") }} -{% endblock %} \ No newline at end of file +{{ vite_asset("apps/github_sync.ts") }} +{% endblock %} diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index 7f635bc96..26c107d34 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -289,62 +289,56 @@ {% endif %} {% endwith %} -{% if codebase.git_mirror or has_change_perm %} -
-
-
-
- - - Github Integration - - beta -
- {% if False %} - {# FIXME: multiple remotes! show all of them #} - {# and link to github configuration page #} - {% if codebase.git_mirror.remote_url %} -
- - - {{ codebase.git_mirror.repo_name }} - - +{% with remotes = codebase.git_mirror.active_remotes if codebase.git_mirror else None %} + {% if has_change_perm or remotes %} +
+
+
+
+ + + Github Sync + + {% if has_change_perm %} + + Configure -
- {% else %} -
- -

- Mirroring in progress. This may take a few seconds. -

- Refresh this page. - -
- {% endif %} - {% elif has_change_perm %} - {% if codebase.live %} -
- {% else %} + {% endif %} +
+ {% if remotes %} +
    + {% for remote in remotes %} +
  • + + {{ remote.repo_name }} + + {% if remote.should_push and remote.should_archive %} + + {% elif remote.should_push %} + + {% elif remote.should_archive %} + + {% endif %} +
    • + {% endfor %} +
+ {% elif has_change_perm and not codebase.live %}
Only available for published models
{% endif %} - {% endif %} -
- - What is this? - +
+ + What is this? + +
-
-{% endif %} + {% endif %} +{% endwith %}
diff --git a/django/library/models.py b/django/library/models.py index 3c15f9b2a..105fe1f82 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -662,6 +662,10 @@ class CodebaseGitRemote(models.Model): date_created = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) + @property + def is_active(self): + return self.should_push or self.should_archive + @property def installation(self) -> GithubIntegrationAppInstallation | None: if self.is_user_repo: @@ -696,7 +700,7 @@ class Meta: models.CheckConstraint( check=models.Q(is_user_repo=True) | models.Q(should_archive=False), name="org_repo_not_archivable", - ) + ), ] @@ -711,6 +715,14 @@ class CodebaseGitMirror(models.Model): date_created = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) + @property + def active_remotes_list(self) -> list[CodebaseGitRemote]: + return list( + self.remotes.filter( + models.Q(should_push=True) | models.Q(should_archive=True) + ) + ) + @property def latest_built_release(self): if not self.built_releases.all(): diff --git a/django/library/serializers.py b/django/library/serializers.py index 208d46504..8dc5e7d00 100644 --- a/django/library/serializers.py +++ b/django/library/serializers.py @@ -506,6 +506,8 @@ class Meta: class CodebaseGitRemoteSerializer(serializers.ModelSerializer): + is_active = serializers.ReadOnlyField(source="is_active") + class Meta: model = CodebaseGitRemote fields = ( @@ -516,6 +518,7 @@ class Meta: "should_push", "should_archive", "is_user_repo", + "is_active", "last_push_log", ) read_only_fields = ( @@ -524,6 +527,7 @@ class Meta: "repo_name", "url", "is_user_repo", + "is_active", "last_push_log", ) diff --git a/django/library/views.py b/django/library/views.py index 52b04f265..3b3ec5fc0 100644 --- a/django/library/views.py +++ b/django/library/views.py @@ -607,7 +607,11 @@ def get_codebase(self): def list(self, request, *args, **kwargs): queryset = self.get_queryset() if request.accepted_renderer.format == "html": - context = {"codebase": self.get_codebase(), "remotes": queryset} + context = { + "codebase": self.get_codebase(), + "remotes": queryset, + "github_org_name": settings.GITHUB_MODEL_LIBRARY_ORG_NAME, + } return Response( context, template_name="library/codebases/git.jinja", @@ -620,14 +624,14 @@ def submitter_installation_status(self, request, *args, **kwargs): codebase = self.get_codebase() submitter = codebase.submitter installation_url = None - try: - social_account = submitter.member_profile.get_social_account("github") + social_account = submitter.member_profile.get_social_account("github") + if social_account: github_account = { "id": social_account.uid, "username": social_account.extra_data.get("login"), "profile_url": social_account.get_profile_url(), } - except SocialAccount.DoesNotExist: + else: github_account = None if github_account: @@ -641,6 +645,7 @@ def submitter_installation_status(self, request, *args, **kwargs): return Response( data={ "github_account": github_account, + "connect_url": reverse("socialaccount_connections"), "installation_url": installation_url, }, status=status.HTTP_200_OK, diff --git a/frontend/src/apps/github_mirror.ts b/frontend/src/apps/github_mirror.ts deleted file mode 100644 index b7633a76b..000000000 --- a/frontend/src/apps/github_mirror.ts +++ /dev/null @@ -1,8 +0,0 @@ -import "vite/modulepreload-polyfill"; - -import { createApp } from "vue"; -import GithubMirrorModal from "@/components/GithubMirrorModal.vue"; -import { extractDataParams } from "@/util"; - -const props = extractDataParams("github-mirror", ["identifier", "defaultRepoName"]); -createApp(GithubMirrorModal, props).mount("#github-mirror"); diff --git a/frontend/src/apps/github_sync.ts b/frontend/src/apps/github_sync.ts new file mode 100644 index 000000000..0876c73de --- /dev/null +++ b/frontend/src/apps/github_sync.ts @@ -0,0 +1,8 @@ +import "vite/modulepreload-polyfill"; + +import { createApp } from "vue"; +import { extractDataParams } from "@/util"; +import GitHubSyncConfiguration from "@/components/GitHubSyncConfiguration.vue"; + +const props = extractDataParams("github-sync", ["codebaseIdentifier", "githubOrgName"]); +createApp(GitHubSyncConfiguration, props).mount("#github-sync"); diff --git a/frontend/src/components/GitHubRemoteItem.vue b/frontend/src/components/GitHubRemoteItem.vue new file mode 100644 index 000000000..8d860c4ce --- /dev/null +++ b/frontend/src/components/GitHubRemoteItem.vue @@ -0,0 +1,13 @@ + + + diff --git a/frontend/src/components/GitHubSyncConfiguration.vue b/frontend/src/components/GitHubSyncConfiguration.vue new file mode 100644 index 000000000..b89c631fd --- /dev/null +++ b/frontend/src/components/GitHubSyncConfiguration.vue @@ -0,0 +1,147 @@ + + + diff --git a/frontend/src/components/GithubMirrorModal.vue b/frontend/src/components/GithubMirrorModal.vue deleted file mode 100644 index 07c671c7b..000000000 --- a/frontend/src/components/GithubMirrorModal.vue +++ /dev/null @@ -1,93 +0,0 @@ - - - diff --git a/frontend/src/composables/api/codebase.ts b/frontend/src/composables/api/codebase.ts index 313cc303d..e5309db98 100644 --- a/frontend/src/composables/api/codebase.ts +++ b/frontend/src/composables/api/codebase.ts @@ -49,10 +49,6 @@ export function useCodebaseAPI() { return del(detailUrl(identifier, ["media", "clear"])); } - async function githubMirror(identifier: string, repoName: string, options?: RequestOptions) { - return post(detailUrl(identifier, ["github_mirror"]), { repoName }, options); - } - return { ...toRefs(state), create, @@ -63,7 +59,6 @@ export function useCodebaseAPI() { mediaClear, mediaListUrl, detailUrl, - githubMirror, searchUrl: searchUrl, }; } diff --git a/frontend/src/composables/api/git.ts b/frontend/src/composables/api/git.ts index 45bf7eaba..b45cc29b9 100644 --- a/frontend/src/composables/api/git.ts +++ b/frontend/src/composables/api/git.ts @@ -1,6 +1,10 @@ import { toRefs } from "vue"; import { useAxios, joinPaths, type RequestOptions } from "@/composables/api"; -import type { CodebaseGitRemote, CodebaseGitRemoteForm, GitIntegrationStatus } from "@/types"; +import type { + CodebaseGitRemote, + CodebaseGitRemoteForm, + GitHubAppInstallationStatus, +} from "@/types"; import type { AxiosResponse } from "axios"; export function useGitRemotesAPI(codebaseIdentifier: string) { @@ -17,9 +21,9 @@ export function useGitRemotesAPI(codebaseIdentifier: string) { return joinPaths([baseUrl, ...paths]); } - async function submitterInstallationStatus( + async function getSubmitterInstallationStatus( options?: RequestOptions - ): Promise> { + ): Promise> { return get(url(["submitter_installation_status"]), options); } @@ -44,7 +48,7 @@ export function useGitRemotesAPI(codebaseIdentifier: string) { detailUrl, list, update, - submitterInstallationStatus, + getSubmitterInstallationStatus, setupOrgGithubRemote, setupUserGithubRemote, }; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index e93db1a53..43038f6d5 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -410,8 +410,9 @@ export interface GithubAccount { installationId?: number; } -export interface GitIntegrationStatus { +export interface GitHubAppInstallationStatus { githubAccount: GithubAccount | null; + connectUrl: string; installationUrl: string | null; } @@ -423,6 +424,7 @@ export interface CodebaseGitRemote { shouldPush: boolean; shouldArchive: boolean; isUserRepo: boolean; + isActive: boolean; lastPushLog: string; } From f4531b7d4d8d00d76a5f218b87939e6392a65510 Mon Sep 17 00:00:00 2001 From: sgfost Date: Wed, 29 Jan 2025 15:41:23 -0700 Subject: [PATCH 08/66] feat: add create/update functionality to the gh sync config --- .../jinja2/library/codebases/git.jinja | 1 + frontend/src/apps/github_sync.ts | 6 +- .../src/components/GitHubRemoteCreateForm.vue | 58 +++++++++ frontend/src/components/GitHubRemoteItem.vue | 98 +++++++++++++- .../components/GitHubSyncConfiguration.vue | 121 ++++++++++++++---- 5 files changed, 252 insertions(+), 32 deletions(-) create mode 100644 frontend/src/components/GitHubRemoteCreateForm.vue diff --git a/django/library/jinja2/library/codebases/git.jinja b/django/library/jinja2/library/codebases/git.jinja index 6036ed4f7..a749e90a6 100644 --- a/django/library/jinja2/library/codebases/git.jinja +++ b/django/library/jinja2/library/codebases/git.jinja @@ -19,6 +19,7 @@ id="github-sync" data-codebase-identifier="{{ codebase.identifier }}" data-github-org-name="{{ github_org_name }}" + data-default-repo-name="{{ slugify(codebase.title) }}" >
{% endblock %} diff --git a/frontend/src/apps/github_sync.ts b/frontend/src/apps/github_sync.ts index 0876c73de..b35ce18ad 100644 --- a/frontend/src/apps/github_sync.ts +++ b/frontend/src/apps/github_sync.ts @@ -4,5 +4,9 @@ import { createApp } from "vue"; import { extractDataParams } from "@/util"; import GitHubSyncConfiguration from "@/components/GitHubSyncConfiguration.vue"; -const props = extractDataParams("github-sync", ["codebaseIdentifier", "githubOrgName"]); +const props = extractDataParams("github-sync", [ + "codebaseIdentifier", + "githubOrgName", + "defaultRepoName", +]); createApp(GitHubSyncConfiguration, props).mount("#github-sync"); diff --git a/frontend/src/components/GitHubRemoteCreateForm.vue b/frontend/src/components/GitHubRemoteCreateForm.vue new file mode 100644 index 000000000..6ccf20517 --- /dev/null +++ b/frontend/src/components/GitHubRemoteCreateForm.vue @@ -0,0 +1,58 @@ + + + diff --git a/frontend/src/components/GitHubRemoteItem.vue b/frontend/src/components/GitHubRemoteItem.vue index 8d860c4ce..4faf41ad4 100644 --- a/frontend/src/components/GitHubRemoteItem.vue +++ b/frontend/src/components/GitHubRemoteItem.vue @@ -1,13 +1,105 @@ diff --git a/frontend/src/components/GitHubSyncConfiguration.vue b/frontend/src/components/GitHubSyncConfiguration.vue index b89c631fd..69860128b 100644 --- a/frontend/src/components/GitHubSyncConfiguration.vue +++ b/frontend/src/components/GitHubSyncConfiguration.vue @@ -1,15 +1,32 @@ @@ -90,29 +157,25 @@ diff --git a/frontend/src/components/GitHubSetupOrgRemoteWizard.vue b/frontend/src/components/GitHubSetupOrgRemoteWizard.vue new file mode 100644 index 000000000..653b27492 --- /dev/null +++ b/frontend/src/components/GitHubSetupOrgRemoteWizard.vue @@ -0,0 +1,76 @@ + + + diff --git a/frontend/src/components/GitHubSetupUserRemoteWizard.vue b/frontend/src/components/GitHubSetupUserRemoteWizard.vue new file mode 100644 index 000000000..6607a5d8b --- /dev/null +++ b/frontend/src/components/GitHubSetupUserRemoteWizard.vue @@ -0,0 +1,130 @@ + + + diff --git a/frontend/src/components/GitHubSyncConfiguration.vue b/frontend/src/components/GitHubSyncConfiguration.vue index 0d23645a9..38d1a78a1 100644 --- a/frontend/src/components/GitHubSyncConfiguration.vue +++ b/frontend/src/components/GitHubSyncConfiguration.vue @@ -51,21 +51,12 @@
-
- - -
+

Repositories on your GitHub account

@@ -115,52 +106,21 @@
-
-
-
- Before you can create a new repository: -
-
    -
  1. - Connect your GitHub account -
    2. -
  2. - - Connect your GitHub account - -
    3. -
  3. - Install the Sync app on your GitHub account -
    4. -
  4. - - Install the Sync app on your GitHub account - -
    5. -
-
- - -
+ +
@@ -170,7 +130,8 @@ import { ref, computed, onMounted } from "vue"; import { useGitRemotesAPI } from "@/composables/api/git"; import type { GitHubAppInstallationStatus, CodebaseGitRemote } from "@/types"; import GitHubRemoteItem from "@/components/GitHubRemoteItem.vue"; -import GitHubRemoteCreateForm from "./GitHubRemoteCreateForm.vue"; +import GitHubSetupOrgRemoteWizard from "@/components/GitHubSetupOrgRemoteWizard.vue"; +import GitHubSetupUserRemoteWizard from "@/components/GitHubSetupUserRemoteWizard.vue"; const props = defineProps<{ codebaseIdentifier: string; @@ -185,12 +146,6 @@ const installationStatus = ref({ installationUrl: null, connectUrl: "", }); -const githubUsername = computed(() => installationStatus.value.githubAccount?.username); -const githubAppInstalled = computed(() => !!installationStatus.value.githubAccount?.installationId); -const canCreateUserRemote = computed(() => !!githubUsername.value && githubAppInstalled.value); - -const showOrgRemoteForm = ref(false); -const showUserRemoteForm = ref(false); const remotesLoading = ref(true); const remotes = ref([]); diff --git a/frontend/src/composables/api/git.ts b/frontend/src/composables/api/git.ts index b45cc29b9..350e98112 100644 --- a/frontend/src/composables/api/git.ts +++ b/frontend/src/composables/api/git.ts @@ -43,6 +43,10 @@ export function useGitRemotesAPI(codebaseIdentifier: string) { return post(url(["setup_user_github_remote"]), { repoName }, options); } + async function setupUserExistingGithubRemote(repoName: string, options?: RequestOptions) { + return post(url(["setup_user_existing_github_remote"]), { repoName }, options); + } + return { ...toRefs(state), detailUrl, @@ -51,5 +55,6 @@ export function useGitRemotesAPI(codebaseIdentifier: string) { getSubmitterInstallationStatus, setupOrgGithubRemote, setupUserGithubRemote, + setupUserExistingGithubRemote, }; } From bf3e88b4c8fc1ea888afac754555f8d78e6d3b79 Mon Sep 17 00:00:00 2001 From: sgfost Date: Fri, 7 Feb 2025 13:04:23 -0700 Subject: [PATCH 13/66] feat: user-owned repos are working allows setting up a push/archive sync that will automatically have the generated git repo pushed to by providing an empty repo, as well as setting up an archive only sync by providing any github repo in both cases, the submitter needs to: - link their github account with the regular oauth flow (so we have a way to match users with a github account) - install the provided 'GitHub Sync' app on their github account with access to any repository that will be synced --- django/library/github_integration.py | 121 ++++++----- ...alter_codebasegitremote_unique_together.py | 17 ++ django/library/models.py | 62 ++---- django/library/serializers.py | 4 + django/library/views.py | 190 +++++++++++++----- frontend/src/components/GitHubRemoteItem.vue | 2 +- .../components/GitHubSetupOrgRemoteWizard.vue | 7 +- .../GitHubSetupUserRemoteWizard.vue | 45 +++-- frontend/src/types.ts | 2 + 9 files changed, 267 insertions(+), 183 deletions(-) create mode 100644 django/library/migrations/0037_alter_codebasegitremote_unique_together.py diff --git a/django/library/github_integration.py b/django/library/github_integration.py index ffcc81ffb..10b57a714 100644 --- a/django/library/github_integration.py +++ b/django/library/github_integration.py @@ -1,8 +1,7 @@ import re from github import GithubIntegration, Auth, Github -from github.GithubException import UnknownObjectException +from github.GithubException import GithubException, UnknownObjectException from github.Repository import Repository as GithubRepo -from github.AuthenticatedUser import AuthenticatedUser from git import Repo as GitRepo from django.conf import settings from django.core.cache import cache @@ -13,67 +12,64 @@ INSTALLATION_ACCESS_TOKEN_REDIS_KEY = "github_installation_access_token" -class GithubRepoNameValidator: - @classmethod - def validate( - cls, - repo_name: str, - installation: GithubIntegrationAppInstallation | None = None, - ): - cls._validate_format(repo_name) - if installation: - cls._check_user_repo_name_unused(repo_name, installation) - else: - cls._check_org_repo_name_unused(repo_name) +class GitHubRepoValidator: - @staticmethod - def _validate_format(repo_name: str): - if not re.fullmatch(r"[A-Za-z0-9_.-]+", repo_name): + def __init__(self, repo_name: str): + self.repo_name = repo_name + + def validate_format(self): + if not re.fullmatch(r"[A-Za-z0-9_.-]+", self.repo_name): raise ValueError( "The repository name can only contain ASCII letters, digits, and the characters ., -, and _" ) - if not (1 <= len(repo_name) <= 100): + if not (1 <= len(self.repo_name) <= 100): raise ValueError("Repository name is too long (maximum is 100 characters)") - if repo_name.endswith(".git"): + if self.repo_name.endswith(".git"): raise ValueError("Repository name cannot end with '.git'") - if "github" in repo_name: + if "github" in self.repo_name: raise ValueError("Repository name cannot contain 'github'") - @staticmethod - def _check_user_repo_name_unused( - repo_name: str, installation: GithubIntegrationAppInstallation - ): - username = installation.github_login - if username in repo_name: - raise ValueError( - f"Repository name cannot contain your username: '{username}'" - ) - github = Github(GithubApi.get_user_installation_access_token(installation)) - try: - github.get_user().get_repo(repo_name) - raise ValueError( - f"Repository name already exists at https://github.com/{username}/{repo_name}" - ) - except UnknownObjectException: - return True - - @staticmethod - def _check_org_repo_name_unused(repo_name: str): - if settings.GITHUB_MODEL_LIBRARY_ORG_NAME in repo_name: + def check_org_repo_name_unused(self): + if settings.GITHUB_MODEL_LIBRARY_ORG_NAME in self.repo_name: raise ValueError( f"Repository name cannot contain the organization name: '{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}'" ) github = Github(GithubApi.get_org_installation_access_token()) + full_name = f"{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}/{self.repo_name}" try: github.get_organization(settings.GITHUB_MODEL_LIBRARY_ORG_NAME).get_repo( - repo_name + self.repo_name ) raise ValueError( - f"Repository name already exists at https://github.com/{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}/{repo_name}" + f"Repository already exists at https://github.com/{full_name}" ) except UnknownObjectException: return True + def get_existing_user_repo_url(self, installation: GithubIntegrationAppInstallation): + token = GithubApi.get_user_installation_access_token(installation) + full_name = f"{installation.github_login}/{self.repo_name}" + github_repo = GithubApi.get_existing_repo(token, full_name) + return github_repo.html_url + + def check_user_repo_empty(self, installation: GithubIntegrationAppInstallation): + token = GithubApi.get_user_installation_access_token(installation) + full_name = f"{installation.github_login}/{self.repo_name}" + github_repo = GithubApi.get_existing_repo( + token, + full_name, + ) + try: + # this should raise a 404 if the repository is empty + github_repo.get_contents("") + raise ValueError( + f"Repository at https://github.com/{full_name} is not empty" + ) + except GithubException as e: + if e.status == 404: + return True + raise e + class GithubApi: """Functionality for interacting with a remote Github repository @@ -106,10 +102,11 @@ def is_user_repo(self): @property def github_repo(self) -> GithubRepo: if not self._github_repo: - try: - self._github_repo = self._get_existing_repo() - except: - raise ValueError("Github repository not created yet") + full_name = f"{self.repo_owner}/{self.repo_name}" + self._github_repo = self.get_existing_repo( + self.installation_access_token, + full_name, + ) return self._github_repo @property @@ -170,7 +167,7 @@ def get_or_create_repo(self, private=False) -> GithubRepo: return self.github_repo except: if self.is_user_repo: - self._github_repo = self._create_user_repo(private) + raise ValueError("User-owned repositories must be created beforehand") else: self._github_repo = self._create_org_repo(private) return self._github_repo @@ -198,24 +195,16 @@ def create_releases(self, local_repo: GitRepo): prerelease=False, ) - def _get_existing_repo(self): + @staticmethod + def get_existing_repo(access_token: str, full_name: str) -> GithubRepo: """attempt to get an existing repository for the authenticated user or organization""" - github = Github(self.installation_access_token) - return github.get_repo(f"{self.repo_owner}/{self.repo_name}") - - def _create_user_repo(self, private=False): - """create a new repository in the user's account - - this function requires the `repo` scope for the installation access token - """ - token = self.installation_access_token - github = Github(token) - repo = github.get_user().create_repo( - name=self.repo_name, - description=f"Mirror of {self.codebase.permanent_url}", - private=private, - ) - return repo + github = Github(access_token) + try: + return github.get_repo(full_name) + except: + raise ValueError( + f"Github repository https://github.com/{full_name} does not exist or is inaccessible" + ) def _create_org_repo(self, private=False): """create a new repository in the CoMSES model library organization @@ -246,6 +235,8 @@ def _push_to_url(self, local_repo: GitRepo, push_url: str) -> str: for info in result: if info.summary: summaries.append(info.summary) + # FIXME: gitpython summaries do not seem to be working as expected + # even when the push succeeds we always get "push failed entirely" if not summaries: return "push failed entirely" return "\n".join(summaries) diff --git a/django/library/migrations/0037_alter_codebasegitremote_unique_together.py b/django/library/migrations/0037_alter_codebasegitremote_unique_together.py new file mode 100644 index 000000000..be5ef90c4 --- /dev/null +++ b/django/library/migrations/0037_alter_codebasegitremote_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.17 on 2025-02-07 19:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("library", "0036_codebasegitremote_is_preexisting_and_more"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="codebasegitremote", + unique_together={("owner", "repo_name")}, + ), + ] diff --git a/django/library/models.py b/django/library/models.py index 2101ff087..6550ec4cb 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -682,11 +682,7 @@ def __str__(self): return f"{self.owner}/{self.repo_name} (active={self.should_push}) (sync={self.should_archive})" class Meta: - unique_together = [ - ("mirror", "owner"), - ("owner", "repo_name"), - ] - unique_together = ("mirror", "owner") + unique_together = ("owner", "repo_name") constraints = [ models.UniqueConstraint( fields=["mirror"], @@ -754,47 +750,21 @@ def mark_releases_built(self, new_releases: models.QuerySet | list): def create_remote(self, owner, repo_name, **kwargs): """create a new remote for this mirror, or get an existing one if there is a matching remote that failed to actually be created at the remote location""" - try: - existing_remote = CodebaseGitRemote.objects.filter( - mirror=self, - owner=owner, - repo_name=repo_name, - url__isnull=False, # proxy for a successful remote creation - ).first() - if existing_remote: - return existing_remote - remote = CodebaseGitRemote.objects.create( - mirror=self, - owner=owner, - repo_name=repo_name, - **kwargs, - ) - return remote - except IntegrityError as e: - if "single_pushable_user_repo" in str(e): - raise ValidationError( - "There can only be one active user-owned repository." - ) - if "single_pushable_org_repo" in str(e): - raise ValidationError( - "There can only be one active organization-owned repository." - ) - if "single_archivable_repo" in str(e): - raise ValidationError( - "There can only be one repository that releases are being archived from." - ) - if "org_repo_not_archivable" in str(e): - raise ValidationError( - "Repositories in the CoMSES Model Library organization cannot be archived." - ) - if "preexisting_repo_not_pushable" in str(e): - raise ValidationError( - "Pre-existing linked repositories cannot be pushed to." - ) - else: - raise ValidationError("A repository already exists at this location.") - - + existing_remote = CodebaseGitRemote.objects.filter( + mirror=self, + owner=owner, + repo_name=repo_name, + url__isnull=True, # proxy for a successful remote creation + ).first() + if existing_remote: + return existing_remote + remote = CodebaseGitRemote.objects.create( + mirror=self, + owner=owner, + repo_name=repo_name, + **kwargs, + ) + return remote @add_to_comses_permission_whitelist class Codebase(index.Indexed, ModeratedContent, ClusterableModel): """ diff --git a/django/library/serializers.py b/django/library/serializers.py index 55ca6c8e0..d5c9f922e 100644 --- a/django/library/serializers.py +++ b/django/library/serializers.py @@ -518,8 +518,10 @@ class Meta: "should_push", "should_archive", "is_user_repo", + "is_preexisting", "is_active", "last_push_log", + "last_archive_log", ) read_only_fields = ( "id", @@ -527,8 +529,10 @@ class Meta: "repo_name", "url", "is_user_repo", + "is_preexisting", "is_active", "last_push_log", + "last_archive_log", ) diff --git a/django/library/views.py b/django/library/views.py index a28ea55b5..534d3ed02 100644 --- a/django/library/views.py +++ b/django/library/views.py @@ -7,7 +7,7 @@ from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin from django.core.exceptions import PermissionDenied -from django.db import transaction +from django.db import transaction, IntegrityError from django.http import HttpResponse, Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404, redirect from django.urls import resolve, reverse @@ -52,7 +52,7 @@ ) from core.pagination import SmallResultSetPagination from core.serializers import RelatedMemberProfileSerializer -from .github_integration import GithubRepoNameValidator +from .github_integration import GitHubRepoValidator from .forms import ( PeerReviewerFeedbackReviewerForm, PeerReviewInvitationReplyForm, @@ -608,6 +608,39 @@ def get_codebase(self): except Codebase.DoesNotExist: raise NotFound("Codebase not found.") + def _forward_integrity_error(self, e: IntegrityError): + msg = str(e) + if "single_pushable_user_repo" in msg: + raise ValidationError( + "There can only be one active user-owned repository." + ) + if "single_pushable_org_repo" in msg: + raise ValidationError( + "There can only be one active organization-owned repository." + ) + if "single_archivable_repo" in msg: + raise ValidationError( + "There can only be one repository that releases are being archived from." + ) + if "org_repo_not_archivable" in msg: + raise ValidationError( + "Repositories in the CoMSES Model Library organization cannot be archived." + ) + if "preexisting_repo_not_pushable" in msg: + raise ValidationError( + "Pre-existing linked repositories cannot be pushed to." + ) + else: + raise ValidationError("A repository already exists at this location.") + + + + def update(self, request, *args, **kwargs): + try: + return super().update(request, *args, **kwargs) + except IntegrityError as e: + self._forward_integrity_error(e) + def list(self, request, *args, **kwargs): queryset = self.get_queryset() if request.accepted_renderer.format == "html": @@ -658,58 +691,104 @@ def submitter_installation_status(self, request, *args, **kwargs): @action(detail=False, methods=["post"]) def setup_user_github_remote(self, request, *args, **kwargs): - # FIXME: this should no longer try to create the repo, instead we need to: - # - check that the repo name given is a bare repo that exists in the user's account - # - push codebase = self.get_codebase() + if not codebase.live: + raise ValidationError("This model does not have any published releases") + installation = codebase.submitter.github_integration_app_installation - return self._setup_github_remote( - codebase=codebase, - repo_name=request.data.get("repo_name"), - owner=installation.github_login, - is_user_repo=True, - should_push=True, - should_archive=True, - installation=installation, + if not installation: + raise ValidationError( + "Installation of the Github integration app is required" + ) + + repo_name = request.data.get("repo_name") + if not repo_name: + raise ValidationError("Repository name is required") + + validator = GitHubRepoValidator(repo_name) + try: + validator.validate_format() + validator.check_user_repo_empty(installation) + except ValueError as e: + raise ValidationError(str(e)) + + # create a mirror if it doesn't exist + if not codebase.git_mirror: + codebase.create_git_mirror() + # set up the user remote + try: + codebase.git_mirror.create_remote( + owner=installation.github_login, + repo_name=repo_name, + is_user_repo=True, + is_preexisting=False, + should_push=True, + should_archive=True, + ) + except IntegrityError as e: + self._forward_integrity_error(e) + # trigger the build and push task + build_and_push_codebase_repo(codebase.id, private_repo=settings.DEBUG) + return Response( + status=status.HTTP_202_ACCEPTED, + data="Synchronization process started, this should take only a few seconds. Refresh this page to see the status.", ) @action(detail=False, methods=["post"]) def setup_user_existing_github_remote(self, request, *args, **kwargs): - # TODO: all this needs to do is setup a remote for archiving - pass - - @action(detail=False, methods=["post"]) - def setup_org_github_remote(self, request, *args, **kwargs): codebase = self.get_codebase() - return self._setup_github_remote( - codebase=codebase, - repo_name=request.data.get("repo_name"), - owner=settings.GITHUB_MODEL_LIBRARY_ORG_NAME, - is_user_repo=False, - should_push=True, - should_archive=False, - ) - - def _setup_github_remote( - self, - codebase: Codebase, - repo_name: str, - owner: str, - is_user_repo: bool, - should_push: bool, - should_archive: bool, - installation=None, - ): - if is_user_repo and not installation: + installation = codebase.submitter.github_integration_app_installation + if not installation: raise ValidationError( "Installation of the Github integration app is required" ) + + repo_name = request.data.get("repo_name") + if not repo_name: + raise ValidationError("Repository name is required") + + validator = GitHubRepoValidator(repo_name) + try: + validator.validate_format() + repo_html_url = validator.get_existing_user_repo_url(installation) + except ValueError as e: + raise ValidationError(str(e)) + + # create a mirror if it doesn't exist + if not codebase.git_mirror: + codebase.create_git_mirror() + # set up the user remote + try: + codebase.git_mirror.create_remote( + owner=installation.github_login, + repo_name=repo_name, + is_user_repo=True, + is_preexisting=True, + should_push=False, + should_archive=True, + url=repo_html_url, + ) + except IntegrityError as e: + self._forward_integrity_error(e) + return Response( + status=status.HTTP_200_OK, + data="Repository remote successfully set up.", + ) + + @action(detail=False, methods=["post"]) + def setup_org_github_remote(self, request, *args, **kwargs): + codebase = self.get_codebase() if not codebase.live: raise ValidationError("This model does not have any published releases") + + repo_name = request.data.get("repo_name") if not repo_name: raise ValidationError("Repository name is required") + + validator = GitHubRepoValidator(repo_name) try: - GithubRepoNameValidator.validate(repo_name, installation=installation) + validator.validate_format() + validator.check_org_repo_name_unused() except ValueError as e: raise ValidationError(str(e)) @@ -717,13 +796,18 @@ def _setup_github_remote( if not codebase.git_mirror: codebase.create_git_mirror() # set up the org remote - codebase.git_mirror.create_remote( - owner=owner, - repo_name=repo_name, - should_push=should_push, - is_user_repo=is_user_repo, - should_archive=should_archive, - ) + try: + codebase.git_mirror.create_remote( + owner=settings.GITHUB_MODEL_LIBRARY_ORG_NAME, + repo_name=repo_name, + is_user_repo=False, + is_preeexisting=False, + should_push=True, + should_archive=False, + ) + except IntegrityError as e: + self._forward_integrity_error(e) + # trigger the build and push task build_and_push_codebase_repo(codebase.id, private_repo=settings.DEBUG) return Response( @@ -761,7 +845,9 @@ def github_sync_webhook(request): sender = payload["sender"] # match based on the uid uid = sender["id"] - social_account = SocialAccount.objects.filter(provider="github", uid=str(uid)).first() + social_account = SocialAccount.objects.filter( + provider="github", uid=str(uid) + ).first() if social_account: repositories = [repo.get("name") for repo in payload["repositories"]] installation = GithubIntegrationAppInstallation.objects.update_or_create( @@ -771,17 +857,21 @@ def github_sync_webhook(request): "github_login": sender["login"], "installation_id": payload["installation"]["id"], "repositories": repositories, - } + }, ) return HttpResponse("OK", status=200) elif event == "release": payload = json.loads(request.body) logger.debug(payload) - if payload["action"] == "released": # release was published, or a pre-release was changed to a release + if ( + payload["action"] == "released" + ): # release was published, or a pre-release was changed to a release # TODO: create a new CodebaseRelease pass - if payload["action"] == "edited": # details of a release, pre-release, or draft were edited + if ( + payload["action"] == "edited" + ): # details of a release, pre-release, or draft were edited # TODO: check if it matches an existing CodebaseRelease that is not published/review_complete # if so, update the CodebaseRelease pass diff --git a/frontend/src/components/GitHubRemoteItem.vue b/frontend/src/components/GitHubRemoteItem.vue index 4faf41ad4..469760882 100644 --- a/frontend/src/components/GitHubRemoteItem.vue +++ b/frontend/src/components/GitHubRemoteItem.vue @@ -6,7 +6,7 @@
-
+
- Create a new repository + Sync to a new repository
@@ -57,7 +57,7 @@ const emit = defineEmits<{ success: []; }>(); -const { isLoading, serverErrors, setupOrgGithubRemote } = useGitRemotesAPI( +const { isLoading, serverErrors, data, setupOrgGithubRemote } = useGitRemotesAPI( props.codebaseIdentifier ); const successMessage = ref(null); @@ -67,8 +67,7 @@ const show = ref(false); const setupRemote = async () => { const onSuccess = () => { - successMessage.value = - "Sync in progress. Refresh this page in a few seconds to check the status."; + successMessage.value = data.value; emit("success"); }; await setupOrgGithubRemote(repoName.value, { onSuccess }); diff --git a/frontend/src/components/GitHubSetupUserRemoteWizard.vue b/frontend/src/components/GitHubSetupUserRemoteWizard.vue index 6607a5d8b..9eb453193 100644 --- a/frontend/src/components/GitHubSetupUserRemoteWizard.vue +++ b/frontend/src/components/GitHubSetupUserRemoteWizard.vue @@ -3,22 +3,23 @@
    -
  1. +
  2. Connect your GitHub account
    3. -
  3. +
  4. Connect your GitHub account
    5. -
  5. +
  6. Install the Sync app on your GitHub account
    7. -
  7. +
  8. Install the Sync app on your GitHub account
    9. -
  9. - Create an empty repository on GitHub +
  10. + + + + Do not initialize the repository + with any files such as a README or license + +
    11. -
  11. +
  12. - +
    {{ githubUsername }}/(); -const { isLoading, serverErrors, setupUserGithubRemote, setupUserExistingGithubRemote } = +const { isLoading, serverErrors, data, setupUserGithubRemote, setupUserExistingGithubRemote } = useGitRemotesAPI(props.codebaseIdentifier); const successMessage = ref(null); const repoName = ref(props.defaultRepoName); @@ -117,8 +129,7 @@ const newRepositoryUrl = computed( const setupRemote = async () => { const onSuccess = () => { - successMessage.value = - "Sync in progress. Refresh this page in a few seconds to check the status."; + successMessage.value = data.value; emit("success"); }; if (props.fromExisting) { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 43038f6d5..18c31362b 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -424,8 +424,10 @@ export interface CodebaseGitRemote { shouldPush: boolean; shouldArchive: boolean; isUserRepo: boolean; + isPreexisting: boolean; isActive: boolean; lastPushLog: string; + lastArchiveLog: string; } export interface CodebaseGitRemoteForm { From 1efdb44dab71ece258a4bd7491341773db248185 Mon Sep 17 00:00:00 2001 From: sgfost Date: Wed, 12 Feb 2025 14:23:24 -0700 Subject: [PATCH 14/66] feat: remove github sync org option, refine UI and overview/help page --- django/library/github_integration.py | 12 +- .../jinja2/library/codebases/git.jinja | 4 +- .../jinja2/library/github_sync_overview.jinja | 149 +++++++++--------- django/library/tasks.py | 8 +- .../GitHubSetupUserRemoteWizard.vue | 4 +- .../components/GitHubSyncConfiguration.vue | 124 ++++++++------- 6 files changed, 157 insertions(+), 144 deletions(-) diff --git a/django/library/github_integration.py b/django/library/github_integration.py index 10b57a714..fb43a4407 100644 --- a/django/library/github_integration.py +++ b/django/library/github_integration.py @@ -34,7 +34,7 @@ def check_org_repo_name_unused(self): raise ValueError( f"Repository name cannot contain the organization name: '{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}'" ) - github = Github(GithubApi.get_org_installation_access_token()) + github = Github(GitHubApi.get_org_installation_access_token()) full_name = f"{settings.GITHUB_MODEL_LIBRARY_ORG_NAME}/{self.repo_name}" try: github.get_organization(settings.GITHUB_MODEL_LIBRARY_ORG_NAME).get_repo( @@ -47,15 +47,15 @@ def check_org_repo_name_unused(self): return True def get_existing_user_repo_url(self, installation: GithubIntegrationAppInstallation): - token = GithubApi.get_user_installation_access_token(installation) + token = GitHubApi.get_user_installation_access_token(installation) full_name = f"{installation.github_login}/{self.repo_name}" - github_repo = GithubApi.get_existing_repo(token, full_name) + github_repo = GitHubApi.get_existing_repo(token, full_name) return github_repo.html_url def check_user_repo_empty(self, installation: GithubIntegrationAppInstallation): - token = GithubApi.get_user_installation_access_token(installation) + token = GitHubApi.get_user_installation_access_token(installation) full_name = f"{installation.github_login}/{self.repo_name}" - github_repo = GithubApi.get_existing_repo( + github_repo = GitHubApi.get_existing_repo( token, full_name, ) @@ -71,7 +71,7 @@ def check_user_repo_empty(self, installation: GithubIntegrationAppInstallation): raise e -class GithubApi: +class GitHubApi: """Functionality for interacting with a remote Github repository and Github API """ diff --git a/django/library/jinja2/library/codebases/git.jinja b/django/library/jinja2/library/codebases/git.jinja index cc8c0f954..4a08e7cab 100644 --- a/django/library/jinja2/library/codebases/git.jinja +++ b/django/library/jinja2/library/codebases/git.jinja @@ -20,12 +20,12 @@

    When creating a new repository here, a git repo will be automatically built from the public - releases of your model and pushed to GitHub. This will be updated every time you + releases of your model and pushed to GitHub. This will be updated every time you publish a new release or update the metadata of an existing one, until it is disabled.

    Changes made to synced repositories on GitHub can be automically pulled back into the CML by - enabling the archiving feature and creating a new release on GitHub. This works + enabling archiving from GitHub and creating a new release on GitHub. This works similarly to the Zenodo GitHub integration.

    diff --git a/django/library/jinja2/library/github_sync_overview.jinja b/django/library/jinja2/library/github_sync_overview.jinja index 993c5fc20..3506b9022 100644 --- a/django/library/jinja2/library/github_sync_overview.jinja +++ b/django/library/jinja2/library/github_sync_overview.jinja @@ -25,64 +25,19 @@

    -

    Hosting Locations

    +

    Sync Directions

    - There are two options for where to host a synced repository, each with its own - advantages and limitations. -

    -
    -
    -
    -

    - {{ github_model_library_org_name }} Organization -

    -
    -

    - Synced repositories in the - - - {{ github_model_library_org_name }} - organization are push-only since they are intended as a read-only mirror of - your model submission on CoMSES. This is a good way to enhance the accessibility of your - model's source code without a GitHub account. -

    -
    Ideal for
    -
      -
    • Adding a convenient, public-facing code browser for your model
      • -
    • Those who don't need or want to develop code on GitHub
      • -
    -
    -
    -
    -
    -
    -

    - Your GitHub Account -

    -
    -

    - Since user-owned repositories give you full control and the ability to make changes, they - can push new CML releases to GitHub, and archive new GitHub - releases back to the CML. This lets you move your model development to GitHub while still - keeping it accessible on CoMSES. -

    -
    Ideal for
    -
      -
    • Taking advantage of tooling to improve your model software or speed up development
      • -
    • Switching to active development on GitHub in order to benefit from the rich collaborative - environment, whether you are a beginner or a seasoned pro
      • -
    -
    -
    -
    -
    -
    -
    -

    Sync Modes

    -

    - Linked repositories can be set up to either Push, Archive, or - both (if user-owned). + The bidirectional sync between the CoMSES Model Library consists of two distinct processes, which can + both be toggled on or off. +

      +
    • + Both processes will be enabled by default when generating a new repository from a model submission. +
      • +
    • + Alternatively, you can link an existing repository with only archiving enabled to import + new releases from GitHub, similarly to the Zenodo GitHub integration. +
      • +

    @@ -185,6 +140,16 @@ in the CoMSES Model Library
    13. +
  13. + +
    + Releases imported from GitHub are initially unpublished in the Model Library, allowing + you to + approve the extracted metadata, make any last-minute changes (by updating the + corresponding GitHub release), + and request peer review. +
    +
  14. @@ -195,9 +160,9 @@
  15. - A codemeta.json file in the root of the repository, along with any - information GitHub has, will be used - to populate the metadata for the release + In order to populate metadata for the release record in the Model Library, + codemeta.json and/or + CITATION.cff files placed in the root of the repository will be parsed
    16. @@ -207,8 +172,8 @@
  16. - Metadata in codemeta.json must be - well-formed to allow the system to properly ingest your GitHub release + Metadata files describing the model software should be well-formed and kept up to date + in the repository to ensure accurate information is imported
  17. @@ -219,18 +184,60 @@ documentation, and other necessary files
-
  • - -
    - Requires linking your personal GitHub account with CoMSES and - installing the GitHub Sync app with permission to administer your repositories, - though you can revoke these permissions or limit them to specific repositories -
    -
    • +
    + {% set faq_entries = { + "What GitHub permissions do you request and why?": "We request read and write access to repository contents. You can + either select specific repositories or allow access to all current and future repositories (though we will not + attempt to access any repositories besides those synced). If you choose the selection option, ensure that the + repository you want to sync is included, which may require updating the selection after creating a blank repository + if you are setting up a new sync from scratch.", + + "Can I sync with a private repository?": "Yes, there are no restrictions on syncing with private repositories.", + + "Can releases imported to the Model Library from GitHub be peer reviewed?": "Yes. Imported releases will be + unpublished initially, allowing you to request a peer review or publish them. However, once a release is published, + you can no longer request a review. While under peer review, the release remains private. If changes are requested, + you must push changes to GitHub, create a new tag, and update the existing release to reference the new tag.", + + "If I set up archiving for an existing repository, will my existing releases be imported?": "No, only future + releases will be imported. This is to prevent duplicate releases from being created.", + + "I created a GitHub release but don't see it on the CoMSES Model Library side, what went wrong?": "Ensure that the + app is installed with the correct permissions. Also, check that the release is **published** (not a draft or in + pre-release mode). If everything seems correct and it's still not appearing, please contact us.", + + "I set up a new synced repository or published a new release, but I don't see anything on the GitHub repository, + what went wrong?": "If changes were made to a branch on GitHub, the sync process will not overwrite them. Ensure + that the app is installed with the correct permissions. If issues persist, please contact us." + } %} +
    +
    +

    Frequently Asked Questions

    + {% for question, answer in faq_entries.items() %} + {% set entry_id = "entry_" ~ loop.index %} + {% set heading_id = "faqHeading_" ~ loop.index %} +
    + +
    +
    + {{ answer }} +
    +
    +
    + {% endfor %} +
    +
    -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/django/library/tasks.py b/django/library/tasks.py index b48fdc024..a810b0925 100644 --- a/django/library/tasks.py +++ b/django/library/tasks.py @@ -2,7 +2,7 @@ from django.conf import settings from .models import Codebase, CodebaseRelease -from .github_integration import GithubApi +from .github_integration import GitHubApi from .fs import CodebaseGitRepositoryApi import logging @@ -21,7 +21,7 @@ def build_and_push_codebase_repo(codebase_id: int, private_repo=False): local_repo = git_fs_api.update_or_build() for remote in mirror.remotes.filter(should_push=True): - gh_api = GithubApi( + gh_api = GitHubApi( codebase=codebase, local_repo=local_repo, remote=remote, @@ -45,7 +45,7 @@ def add_releases_and_push_codebase_repo(codebase_id: int): local_repo = git_fs_api.append_releases() for remote in mirror.remotes.filter(should_push=True): - gh_api = GithubApi( + gh_api = GitHubApi( codebase=codebase, local_repo=local_repo, remote=remote, @@ -75,7 +75,7 @@ def update_release_branch_and_push_codebase_repo(release_id: int): # local_repo will be None if no changes were made if local_repo: for remote in mirror.remotes.filter(should_push=True): - gh_api = GithubApi( + gh_api = GitHubApi( codebase=codebase, local_repo=local_repo, remote=remote, diff --git a/frontend/src/components/GitHubSetupUserRemoteWizard.vue b/frontend/src/components/GitHubSetupUserRemoteWizard.vue index 9eb453193..9f80ebaec 100644 --- a/frontend/src/components/GitHubSetupUserRemoteWizard.vue +++ b/frontend/src/components/GitHubSetupUserRemoteWizard.vue @@ -3,8 +3,8 @@
      diff --git a/frontend/src/components/GitHubSyncConfiguration.vue b/frontend/src/components/GitHubSyncConfiguration.vue index 38d1a78a1..6c094bea7 100644 --- a/frontend/src/components/GitHubSyncConfiguration.vue +++ b/frontend/src/components/GitHubSyncConfiguration.vue @@ -1,6 +1,7 @@ From 0f8fc0181b9b93bda970587ac4ddaba1d538783e Mon Sep 17 00:00:00 2001 From: sgfost Date: Thu, 13 Feb 2025 10:48:40 -0700 Subject: [PATCH 15/66] fix: rename github -> CML process 'archiving' -> 'importing' and squash migrations "import(ing)" is the wording I keep finding describes the process the best other potential names and their issues: * publish - same name as the direct publishing, releases need to be manually published after they are imported * pull - git command that is not used * fetch - git command that is not used * ingest - ok and similar to import but not quite as clear --- django/curator/fs.py | 2 +- .../jinja2/library/codebases/git.jinja | 2 +- .../library/codebases/releases/retrieve.jinja | 8 +- .../jinja2/library/github_sync_overview.jinja | 12 +- ...mirror_importedreleasepackage_and_more.py} | 123 ++++++++++++++++-- ..._alter_codebasegitremote_owner_and_more.py | 30 ----- .../0035_externalreleasepackage_and_more.py | 96 -------------- ...debasegitremote_is_preexisting_and_more.py | 30 ----- ...alter_codebasegitremote_unique_together.py | 17 --- django/library/models.py | 30 ++--- django/library/serializers.py | 2 +- django/library/views.py | 6 +- frontend/src/components/GitHubRemoteItem.vue | 8 +- .../GitHubSetupUserRemoteWizard.vue | 6 +- .../components/GitHubSyncConfiguration.vue | 71 +--------- frontend/src/types.ts | 4 +- 16 files changed, 153 insertions(+), 294 deletions(-) rename django/library/migrations/{0033_codebasegitmirror_githubintegrationappinstallation_and_more.py => 0033_codebasegitmirror_importedreleasepackage_and_more.py} (56%) delete mode 100644 django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py delete mode 100644 django/library/migrations/0035_externalreleasepackage_and_more.py delete mode 100644 django/library/migrations/0036_codebasegitremote_is_preexisting_and_more.py delete mode 100644 django/library/migrations/0037_alter_codebasegitremote_unique_together.py diff --git a/django/curator/fs.py b/django/curator/fs.py index 71dbdffdf..6a197fda2 100644 --- a/django/curator/fs.py +++ b/django/curator/fs.py @@ -8,7 +8,7 @@ def fsck(queryset): results = OrderedDict() for release in queryset: - if release.is_external_package: + if release.is_imported: continue rfsc = CodebaseReleaseFileConsistencyChecker(release) errors = rfsc.check() diff --git a/django/library/jinja2/library/codebases/git.jinja b/django/library/jinja2/library/codebases/git.jinja index 4a08e7cab..36f5d187f 100644 --- a/django/library/jinja2/library/codebases/git.jinja +++ b/django/library/jinja2/library/codebases/git.jinja @@ -25,7 +25,7 @@

      Changes made to synced repositories on GitHub can be automically pulled back into the CML by - enabling archiving from GitHub and creating a new release on GitHub. This works + enabling importing from GitHub and creating a new release on GitHub. This works similarly to the Zenodo GitHub integration.

      diff --git a/django/library/jinja2/library/codebases/releases/retrieve.jinja b/django/library/jinja2/library/codebases/releases/retrieve.jinja index b4aece036..c5d6a7649 100644 --- a/django/library/jinja2/library/codebases/releases/retrieve.jinja +++ b/django/library/jinja2/library/codebases/releases/retrieve.jinja @@ -314,12 +314,12 @@ {{ remote.owner }}/{{ remote.repo_name }} - {# {% if remote.should_push and remote.should_archive %} - + {# {% if remote.should_push and remote.should_import %} + {% elif remote.should_push %} - {% elif remote.should_archive %} - + {% elif remote.should_import %} + {% endif %} #} {% endfor %} diff --git a/django/library/jinja2/library/github_sync_overview.jinja b/django/library/jinja2/library/github_sync_overview.jinja index 3506b9022..2973c8d13 100644 --- a/django/library/jinja2/library/github_sync_overview.jinja +++ b/django/library/jinja2/library/github_sync_overview.jinja @@ -34,7 +34,7 @@ Both processes will be enabled by default when generating a new repository from a model submission.

    1. - Alternatively, you can link an existing repository with only archiving enabled to import + Alternatively, you can link an existing repository with only importing enabled to import new releases from GitHub, similarly to the Zenodo GitHub integration.
      2. @@ -43,7 +43,7 @@
      -

      Push

      +

      Pushing

      (CoMSES → GitHub)
      @@ -124,7 +124,7 @@
      -

      Archive

      +

      Importing

      (GitHub → CoMSES)
      @@ -136,7 +136,7 @@ Whenever you publish a new release on a linked GitHub repository with - archiving enabled, the release is automatically imported as a new release of your model + importing enabled, the release is automatically imported as a new release of your model in the CoMSES Model Library
      @@ -179,7 +179,7 @@
    2. - Archived releases do not enforce a certain directory structure, instead pulling in the + Releases imported from GitHub do not enforce a certain directory structure, instead pulling in the release assets as-is, so you must ensure this contains all of your source code, documentation, and other necessary files
      @@ -204,7 +204,7 @@ you can no longer request a review. While under peer review, the release remains private. If changes are requested, you must push changes to GitHub, create a new tag, and update the existing release to reference the new tag.", - "If I set up archiving for an existing repository, will my existing releases be imported?": "No, only future + "If I set up importing for an existing repository, will my existing releases be imported?": "No, only future releases will be imported. This is to prevent duplicate releases from being created.", "I created a GitHub release but don't see it on the CoMSES Model Library side, what went wrong?": "Ensure that the diff --git a/django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py b/django/library/migrations/0033_codebasegitmirror_importedreleasepackage_and_more.py similarity index 56% rename from django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py rename to django/library/migrations/0033_codebasegitmirror_importedreleasepackage_and_more.py index a7f8db4ec..894b060c3 100644 --- a/django/library/migrations/0033_codebasegitmirror_githubintegrationappinstallation_and_more.py +++ b/django/library/migrations/0033_codebasegitmirror_importedreleasepackage_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.17 on 2025-01-25 01:34 +# Generated by Django 4.2.17 on 2025-02-13 17:57 from django.conf import settings from django.db import migrations, models @@ -35,6 +35,74 @@ class Migration(migrations.Migration): ), ], ), + migrations.CreateModel( + name="ImportedReleasePackage", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("date_created", models.DateTimeField(auto_now_add=True)), + ("last_modified", models.DateTimeField(auto_now=True)), + ( + "service", + models.CharField( + choices=[("github", "GitHub")], + help_text="The external service where the release is hosted", + max_length=32, + ), + ), + ( + "uid", + models.CharField( + help_text="Unique identifier for this external release e.g. github release id", + max_length=128, + unique=True, + ), + ), + ( + "name", + models.CharField( + help_text="Name of the release, ideally the tag name if applicable", + max_length=255, + ), + ), + ( + "display_name", + models.CharField( + blank=True, + help_text="A human-readable name for the release, if different from the name", + max_length=255, + ), + ), + ( + "html_url", + models.URLField( + blank=True, + help_text="URL to the (browser-viewable) page for the release on the external service", + ), + ), + ( + "download_url", + models.URLField( + blank=True, + help_text="URL to the downloadable package for the release on the external service, ideally a zipball such as in the case of github", + ), + ), + ( + "extra_data", + models.JSONField( + default=dict, + help_text="Additional data for the external release, ideally the full object (e.g. github release) from the external service", + ), + ), + ], + ), migrations.CreateModel( name="GithubIntegrationAppInstallation", fields=[ @@ -75,32 +143,37 @@ class Migration(migrations.Migration): ), ), ( - "should_push", + "is_user_repo", models.BooleanField( - default=True, help_text="Whether to push to this remote" + default=False, help_text="Whether this is a user-owned remote" ), ), ( - "is_user_repo", + "is_preexisting", models.BooleanField( - default=False, help_text="Whether this is a user-owned remote" + default=False, + help_text="Whether this remote was pre-existing or based on a codebase git mirror", ), ), ( - "should_archive", + "should_push", + models.BooleanField( + default=True, help_text="Whether to push to this remote" + ), + ), + ( + "should_import", models.BooleanField( default=False, - help_text="Whether to archive new releases from this remote", + help_text="Whether to import new releases from this remote", ), ), ("repo_name", models.CharField(max_length=100)), ( "owner", models.CharField( - blank=True, help_text="Github username (or organization) who owns the remote repository", max_length=100, - null=True, ), ), ( @@ -110,6 +183,7 @@ class Migration(migrations.Migration): ), ), ("last_push_log", models.TextField(blank=True)), + ("last_archive_log", models.TextField(blank=True)), ("date_created", models.DateTimeField(auto_now_add=True)), ("last_modified", models.DateTimeField(auto_now=True)), ( @@ -132,6 +206,15 @@ class Migration(migrations.Migration): to="library.codebasegitmirror", ), ), + migrations.AddField( + model_name="codebaserelease", + name="imported_release_package", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="library.importedreleasepackage", + ), + ), migrations.AddConstraint( model_name="codebasegitremote", constraint=models.UniqueConstraint( @@ -151,13 +234,31 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="codebasegitremote", constraint=models.UniqueConstraint( - condition=models.Q(("should_archive", True)), + condition=models.Q(("should_import", True)), fields=("mirror",), name="single_archivable_repo", ), ), + migrations.AddConstraint( + model_name="codebasegitremote", + constraint=models.CheckConstraint( + check=models.Q( + ("is_user_repo", True), ("should_import", False), _connector="OR" + ), + name="org_repo_not_archivable", + ), + ), + migrations.AddConstraint( + model_name="codebasegitremote", + constraint=models.CheckConstraint( + check=models.Q( + ("is_preexisting", False), ("should_push", False), _connector="OR" + ), + name="preexisting_repo_not_pushable", + ), + ), migrations.AlterUniqueTogether( name="codebasegitremote", - unique_together={("mirror", "owner")}, + unique_together={("owner", "repo_name")}, ), ] diff --git a/django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py b/django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py deleted file mode 100644 index 169339c2b..000000000 --- a/django/library/migrations/0034_alter_codebasegitremote_owner_and_more.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 4.2.17 on 2025-01-28 02:16 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("library", "0033_codebasegitmirror_githubintegrationappinstallation_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="codebasegitremote", - name="owner", - field=models.CharField( - help_text="Github username (or organization) who owns the remote repository", - max_length=100, - ), - ), - migrations.AddConstraint( - model_name="codebasegitremote", - constraint=models.CheckConstraint( - check=models.Q( - ("is_user_repo", True), ("should_archive", False), _connector="OR" - ), - name="org_repo_not_archivable", - ), - ), - ] diff --git a/django/library/migrations/0035_externalreleasepackage_and_more.py b/django/library/migrations/0035_externalreleasepackage_and_more.py deleted file mode 100644 index c9c3d7d83..000000000 --- a/django/library/migrations/0035_externalreleasepackage_and_more.py +++ /dev/null @@ -1,96 +0,0 @@ -# Generated by Django 4.2.17 on 2025-02-04 21:49 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ("library", "0034_alter_codebasegitremote_owner_and_more"), - ] - - operations = [ - migrations.CreateModel( - name="ExternalReleasePackage", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("date_created", models.DateTimeField(auto_now_add=True)), - ("last_modified", models.DateTimeField(auto_now=True)), - ( - "service", - models.CharField( - choices=[("github", "GitHub")], - help_text="The external service where the release is hosted", - max_length=32, - ), - ), - ( - "uid", - models.CharField( - help_text="Unique identifier for this external release e.g. github release id", - max_length=128, - unique=True, - ), - ), - ( - "name", - models.CharField( - help_text="Name of the release, ideally the tag name if applicable", - max_length=255, - ), - ), - ( - "display_name", - models.CharField( - blank=True, - help_text="A human-readable name for the release, if different from the name", - max_length=255, - ), - ), - ( - "html_url", - models.URLField( - blank=True, - help_text="URL to the (browser-viewable) page for the release on the external service", - ), - ), - ( - "download_url", - models.URLField( - blank=True, - help_text="URL to the downloadable package for the release on the external service, ideally a zipball such as in the case of github", - ), - ), - ( - "extra_data", - models.JSONField( - default=dict, - help_text="Additional data for the external release, ideally the full object (e.g. github release) from the external service", - ), - ), - ], - ), - migrations.AddField( - model_name="codebasegitremote", - name="last_archive_log", - field=models.TextField(blank=True), - ), - migrations.AddField( - model_name="codebaserelease", - name="external_release_package", - field=models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="library.externalreleasepackage", - ), - ), - ] diff --git a/django/library/migrations/0036_codebasegitremote_is_preexisting_and_more.py b/django/library/migrations/0036_codebasegitremote_is_preexisting_and_more.py deleted file mode 100644 index e569c95f5..000000000 --- a/django/library/migrations/0036_codebasegitremote_is_preexisting_and_more.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 4.2.17 on 2025-02-07 02:21 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("library", "0035_externalreleasepackage_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="codebasegitremote", - name="is_preexisting", - field=models.BooleanField( - default=False, - help_text="Whether this remote was pre-existing or based on a codebase git mirror", - ), - ), - migrations.AddConstraint( - model_name="codebasegitremote", - constraint=models.CheckConstraint( - check=models.Q( - ("is_preexisting", False), ("should_push", False), _connector="OR" - ), - name="preexisting_repo_not_pushable", - ), - ), - ] diff --git a/django/library/migrations/0037_alter_codebasegitremote_unique_together.py b/django/library/migrations/0037_alter_codebasegitremote_unique_together.py deleted file mode 100644 index be5ef90c4..000000000 --- a/django/library/migrations/0037_alter_codebasegitremote_unique_together.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.2.17 on 2025-02-07 19:03 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("library", "0036_codebasegitremote_is_preexisting_and_more"), - ] - - operations = [ - migrations.AlterUniqueTogether( - name="codebasegitremote", - unique_together={("owner", "repo_name")}, - ), - ] diff --git a/django/library/models.py b/django/library/models.py index 6550ec4cb..f84e4ad1d 100644 --- a/django/library/models.py +++ b/django/library/models.py @@ -646,8 +646,8 @@ class CodebaseGitRemote(models.Model): should_push = models.BooleanField( default=True, help_text="Whether to push to this remote" ) - should_archive = models.BooleanField( - default=False, help_text="Whether to archive new releases from this remote" + should_import = models.BooleanField( + default=False, help_text="Whether to import new releases from this remote" ) repo_name = models.CharField(max_length=100) owner = models.CharField( @@ -670,7 +670,7 @@ class CodebaseGitRemote(models.Model): @property def is_active(self): - return self.should_push or self.should_archive + return self.should_push or self.should_import @property def installation(self) -> GithubIntegrationAppInstallation | None: @@ -679,7 +679,7 @@ def installation(self) -> GithubIntegrationAppInstallation | None: return None def __str__(self): - return f"{self.owner}/{self.repo_name} (active={self.should_push}) (sync={self.should_archive})" + return f"{self.owner}/{self.repo_name} (active={self.should_push}) (sync={self.should_import})" class Meta: unique_together = ("owner", "repo_name") @@ -696,11 +696,11 @@ class Meta: ), models.UniqueConstraint( fields=["mirror"], - condition=models.Q(should_archive=True), + condition=models.Q(should_import=True), name="single_archivable_repo", ), models.CheckConstraint( - check=models.Q(is_user_repo=True) | models.Q(should_archive=False), + check=models.Q(is_user_repo=True) | models.Q(should_import=False), name="org_repo_not_archivable", ), models.CheckConstraint( @@ -725,7 +725,7 @@ class CodebaseGitMirror(models.Model): def active_remotes_list(self) -> list[CodebaseGitRemote]: return list( self.remotes.filter( - models.Q(should_push=True) | models.Q(should_archive=True) + models.Q(should_push=True) | models.Q(should_import=True) ) ) @@ -1372,7 +1372,7 @@ class CodebasePublication(models.Model): """ -class ExternalReleasePackage(models.Model): +class ImportedReleasePackage(models.Model): date_created = models.DateTimeField(auto_now_add=True) last_modified = models.DateTimeField(auto_now=True) @@ -1443,11 +1443,11 @@ def with_submitter(self): def internal(self, **kwargs): """returns only releases that are locally uploaded and not archived from an external service""" - return self.filter(external_release_package__isnull=True, **kwargs) + return self.filter(imported_release_package__isnull=True, **kwargs) def external(self, **kwargs): """returns only releases that are archived from an external service such as GitHub""" - return self.filter(external_release_package__isnull=False, **kwargs) + return self.filter(imported_release_package__isnull=False, **kwargs) def public(self, **kwargs): return self.filter(status=CodebaseRelease.Status.PUBLISHED, **kwargs) @@ -1538,8 +1538,8 @@ class Status(models.TextChoices): max_length=32, ) - external_release_package = models.OneToOneField( - ExternalReleasePackage, + imported_release_package = models.OneToOneField( + ImportedReleasePackage, null=True, on_delete=models.SET_NULL, ) @@ -2083,8 +2083,8 @@ def get_status_color(self): return COLOR_MAP.get(self.status) @property - def is_external_package(self): - return self.external_release_package is not None + def is_imported(self): + return self.imported_release_package is not None def create_or_update_codemeta(self, force=True): return self.get_fs_api().create_or_update_codemeta(force=force) @@ -2093,7 +2093,7 @@ def get_fs_api( self, mimetype_mismatch_message_level=MessageLevels.error ) -> CodebaseReleaseFsApi: """Factory method to return the appropriate filesystem API for this release.""" - if self.is_external_package: + if self.is_imported: return ExternalCodebaseReleaseFsApi.initialize(self) return CodebaseReleaseFsApi.initialize( self, mimetype_mismatch_message_level=mimetype_mismatch_message_level diff --git a/django/library/serializers.py b/django/library/serializers.py index d5c9f922e..0d12e7c58 100644 --- a/django/library/serializers.py +++ b/django/library/serializers.py @@ -516,7 +516,7 @@ class Meta: "repo_name", "url", "should_push", - "should_archive", + "should_import", "is_user_repo", "is_preexisting", "is_active", diff --git a/django/library/views.py b/django/library/views.py index 534d3ed02..047a81b44 100644 --- a/django/library/views.py +++ b/django/library/views.py @@ -723,7 +723,7 @@ def setup_user_github_remote(self, request, *args, **kwargs): is_user_repo=True, is_preexisting=False, should_push=True, - should_archive=True, + should_import=True, ) except IntegrityError as e: self._forward_integrity_error(e) @@ -765,7 +765,7 @@ def setup_user_existing_github_remote(self, request, *args, **kwargs): is_user_repo=True, is_preexisting=True, should_push=False, - should_archive=True, + should_import=True, url=repo_html_url, ) except IntegrityError as e: @@ -803,7 +803,7 @@ def setup_org_github_remote(self, request, *args, **kwargs): is_user_repo=False, is_preeexisting=False, should_push=True, - should_archive=False, + should_import=False, ) except IntegrityError as e: self._forward_integrity_error(e) diff --git a/frontend/src/components/GitHubRemoteItem.vue b/frontend/src/components/GitHubRemoteItem.vue index 469760882..ec43f67db 100644 --- a/frontend/src/components/GitHubRemoteItem.vue +++ b/frontend/src/components/GitHubRemoteItem.vue @@ -26,10 +26,10 @@ class="form-check-input" type="checkbox" id="flexSwitchCheckDefault1" - :checked="remote.shouldArchive" - @change="confirmToggle('shouldArchive', remote.shouldArchive, $event)" + :checked="remote.shouldImport" + @change="confirmToggle('shouldImport', remote.shouldImport, $event)" /> - +
      from GitHub