+
+ {% endif %}
+ {% endwith %}
+{% endwith %}
+
+
+
+
+
+
+ GitHub
+ {% if github_config.is_beta %}
+ Beta
+ {% endif %}
+
+
+
+
+
+ {% if remote %}
+
+ {{ remote.owner }}/{{ remote.repo_name }}
+
+ {% endif %}
+ {% if has_change_perm %}
+
+
+ {{ "Manage" if remote else "Connect a repository" }}
+
+ {% endif %}
+
@@ -307,11 +347,11 @@
- {% if codebase.repository_url %}
+ {% if codebase.repository_url and not remote %}
External Repository
-
-
+
+
{{ strip_url_scheme(codebase.repository_url) }}
@@ -320,8 +360,8 @@
{% if release.output_data_url %}
Output Data
-
-
+
+
{{ strip_url_scheme(release.output_data_url) }}
@@ -331,12 +371,10 @@
DOI
{% if release.doi %}
-
+ {{ release.doi }}
{% elif codebase.doi %}
This DOI is for the parent Codebase and always points at the most recent published version of this model. - {{ codebase.doi }}
+ {{ codebase.doi }} {% else %}No assigned DOI
{% if has_change_perm %}
@@ -356,9 +394,24 @@
{{ release.version_number }}
+ {% with imported_sync_state = release.imported_release_sync_state %}
+ {% if imported_sync_state and imported_sync_state.display_name and imported_sync_state.html_url %}
+ Imported From
+
+ {% endif %}
+ {% endwith %}
License
- {{ release.license.name }}
+ {% if release.license %}
+ {{ release.license.name }}
+ {% else %}
+ None
+ {% endif %}
Programming Language
@@ -428,7 +481,7 @@
-
+
@@ -508,7 +561,7 @@
{% for related_release in codebase.ordered_releases_list(has_change_perm, asc=False) %}
@@ -508,7 +561,7 @@
{% for related_release in codebase.ordered_releases_list(has_change_perm, asc=False) %}
- {{ related_release.version_number }}
+ {{ related_release.version_number }} {{ imported_release_indicator(related_release) }}
{% if release.live or has_change_perm %}
{{ member_profile_href(related_release.submitter.member_profile) }}
diff --git a/django/library/jinja2/library/github_overview.jinja b/django/library/jinja2/library/github_overview.jinja
new file mode 100644
index 000000000..08d4981d9
--- /dev/null
+++ b/django/library/jinja2/library/github_overview.jinja
@@ -0,0 +1,190 @@
+{% extends "base.jinja" %}
+
+{% block introduction %}
+{% with github_config=settings("library.GitHubIntegrationConfiguration", use_default_site=True) %}
+GitHub Integration Overview{% if github_config.is_beta %} Beta{% endif %}
+{% endwith %}
+{% endblock %}
+
+{% block content %}
+{% with github_config=settings("library.GitHubIntegrationConfiguration", use_default_site=True) %}
+{% macro icon_bullets(items) -%}
+
+ {%- for item in items %}
+ {% set icon = item.icon or "fas fa-chevron-right" %}
+ -
+
+ {{ item.text.rendered|safe }}
+
+ {%- endfor %}
+
+{%- endmacro %}
+
+{% macro tutorial_videos(videos) -%}
+{%- for video in videos %}
+
+ {% if video.title %}
+ {{ video.title }}
+ {% endif %}
+
+ {% if video.caption and video.caption.raw %}
+ {{ video.caption.rendered|safe }}
+ {% endif %}
+
+{%- endfor %}
+{%- endmacro %}
+
+{% macro faq_accordion(entries, accordion_id="github-faq-accordion") -%}
+
+ {% for entry in entries %}
+ {% set entry_id = accordion_id ~ "_entry_" ~ loop.index %}
+ {% set heading_id = accordion_id ~ "_heading_" ~ loop.index %}
+
+
+
+
+
+
+
+
+ {{ entry.answer.rendered|safe }}
+
+
+
+ {% endfor %}
+
+{%- endmacro %}
+
+
+
+
+
+
+ The CoMSES Model Library GitHub integration
+ allows you to transfer releases of your model to or from a GitHub
+ repository by connecting an existing repository or by having one created for you.
+
+
+ Get started by pressing
+
+ Connect a repository
+
+ for one of your models listed below or on the model's
+ page. You can also import directly from an existing GitHub repository when submitting a new model by selecting
+ Import model from GitHub
+ on the new model form.
+ Once you have a connected repository, it will be linked on the model's page and
+ visible to others.
+
+
+ {% if user.is_authenticated %}
+
+ {% else %}
+
+ You must be signed
+ in to manage the GitHub integration for your models.
+
+ {% endif %}
+
+ Learn more about how it works
+
+
+
+ Import model code from GitHub
+ Have an existing GitHub repository for your model? Connect
+ it to import releases straight into the CoMSES Model Library.
+
+
+ Read more
+
+
+
+
+
+
+
+
+ Set up a new GitHub repository
+ Don't have a repository yet? We'll build a git repository
+ from your existing releases and let you push it to your own GitHub account.
+
+
+ Read more
+
+
+
+
+
+
+
+
+ Have questions? Concerns?
+ Check out the list of frequently ask questions. If you
+ don't find an answer, we are happy to help, just contact us.
+
+
+ FAQs
+
+
+
+
+
+
+
+
+
+ Import models from an existing GitHub repository
+
+
+ {% if github_config %}
+ {{ tutorial_videos(github_config.existing_repo_tutorial_videos.all()) }}
+ {% endif %}
+
+ {% if github_config %}
+ {{ icon_bullets(github_config.existing_repo_info_items.all()) }}
+ {% endif %}
+
+
+
+
+
+
+ Set up a new GitHub repository for your submitted model
+
+
+ {% if github_config %}
+ {{ tutorial_videos(github_config.new_repo_tutorial_videos.all()) }}
+ {% endif %}
+
+ {% if github_config %}
+ {{ icon_bullets(github_config.new_repo_info_items.all()) }}
+ {% endif %}
+
+
+
+
+
+
+ Frequently asked questions
+ {% if github_config and github_config.faq_entries.exists() %}
+ {{ faq_accordion(github_config.faq_entries.all()) }}
+ {% endif %}
+
+
+
+
+
+{% endwith %}
+{% endblock %}
+
+{% block js %}
+{{ vite_asset("apps/github_overview.ts") }}
+{% endblock %}
\ No newline at end of file
diff --git a/django/library/jinja2/library/includes/publish_model_help.jinja b/django/library/jinja2/library/includes/publish_model_help.jinja
index 24dc87e18..7fdf3900e 100644
--- a/django/library/jinja2/library/includes/publish_model_help.jinja
+++ b/django/library/jinja2/library/includes/publish_model_help.jinja
@@ -1,26 +1,47 @@
-
-
- About the CoMSES Model Library
-
- more info
+
+
+
+ About the Model Library
-
-
-
-
- Our mission is to help computational modelers develop, document, and share their computational models in accordance with community standards and good open science and software engineering practices. Model authors can publish their model source code in the Computational Model Library with narrative documentation as well as metadata that supports open science and emerging norms that facilitate software citation, computational reproducibility / frictionless reuse, and interoperability. Model authors can also request private peer review of their computational models. Models that pass peer review receive a DOI once published.
-
-
- All users of models published in the library must cite model authors when they use and benefit from their code.
-
+
+ Publishing Tutorial
+
+ {% with github_config=settings("library.GitHubIntegrationConfiguration", use_default_site=True) %}
+ {% if github_config.can_use_github_integration(user) %}
+
+ GitHub
+
+ {% endif %}
+ {% endwith %}
+
+ Peer Review & DOIs
+
+
+
+
+
+ Our mission is to help computational modelers develop, document, and share their computational models in
+ accordance with community standards and
+ good open science and software engineering
+ practices. Model authors can publish their model source code in the Computational Model Library with narrative documentation as well as metadata
+ that supports open science and emerging norms that facilitate software citation, computational
+ reproducibility / frictionless reuse, and interoperability. Model authors can also request private peer review of their computational models. Models that pass peer review
+ receive a DOI once published.
+
+
+ All users of models published in the library must cite model authors when they use and benefit
+ from their code.
+
+
+ Please check out our model publishing
+ tutorial and feel free to contact us if you have any
+ questions or concerns about publishing your model(s) in the Computational Model Library.
+
+
-
- Please check out our model publishing tutorial and feel free to contact us if you have any questions or concerns about publishing your model(s) in the Computational Model Library.
-
-
- We also maintain a curated database of over 7500 publications of
- agent-based and individual based models with detailed metadata on availability of code and
- bibliometric information on the landscape of ABM/IBM publications that we welcome you to explore.
-
-
+
\ No newline at end of file
diff --git a/django/library/jinja2/library/review/dashboard.jinja b/django/library/jinja2/library/review/dashboard.jinja
index 9514800bd..77e6fcec7 100644
--- a/django/library/jinja2/library/review/dashboard.jinja
+++ b/django/library/jinja2/library/review/dashboard.jinja
@@ -1,6 +1,7 @@
{% extends 'sidebar_layout.jinja' %}
{% from "common.jinja" import breadcrumb, paginator %}
{% from "library/review/includes/macros.jinja" import display_closed_status %}
+{% from "library/codebases/macros.jinja" import imported_release_indicator %}
{% block introduction %}
Peer Review Editor Dashboard
@@ -32,6 +33,7 @@
{% set review = release.review %}
+ {{ imported_release_indicator(release) }}
Review for version {{ release.version_number }}
{{ display_closed_status(review) }}
diff --git a/django/library/jinja2/library/review/email/model_revisions_requested_imported.jinja b/django/library/jinja2/library/review/email/model_revisions_requested_imported.jinja
new file mode 100644
index 000000000..8364ca9aa
--- /dev/null
+++ b/django/library/jinja2/library/review/email/model_revisions_requested_imported.jinja
@@ -0,0 +1,23 @@
+{% set release=review.codebase_release %}
+{% set imported_sync_state = release.imported_release_sync_state %}
+
+Dear {{ review.contact_author_name }},
+
+The reviewer for your computational model [{{ release.title }}]({{ build_absolute_uri(release.share_url) }}) has recommended that changes be made to your model. These are summarized below:
+
+{% autoescape false %}
+{{ feedback.notes_to_author }}
+{% endautoescape %}
+
+Since this release is imported from GitHub, you will need to update it there and then re-import it into CoMSES Net. Here is the process:
+
+1. **Commit and push your changes:** Make the necessary changes to your code and push them to your GitHub repository.
+2. **Edit the release on GitHub:** Go to the [release page on GitHub]({{ imported_sync_state.html_url }}) to edit the release.
+3. **Update the release tag:**
+ - In the release edit form, you can choose which tag or branch the release should be associated with.
+ - To include your latest changes, you should create a new tag. You can do this by typing a new version number in the "Tag version" box and selecting "Create new tag: [new_version] on publish". Make sure this new tag points to the branch with your recent commits.
+ - Keep in mind that the original version number will be kept, so its recommended to append a suffix to the tag instead of incrementing, e.g. "v1.0.0-rev1"
+4. **Confirm the release has been updated:** Follow [this link]({{ build_absolute_uri(release.get_edit_url()) }}) to confirm that the release has been updated, making sure files are categorized and metadata is complete.
+5. **Notify reviewer:** Once changes have been made, use the "Notify Reviewer of Changes" button to send an alert to the reviewer.
+
+On behalf of the editors@comses.net, thank you for submitting your computational model(s) to CoMSES Net! Our peer review service is intended to serve the community and we hope that you find the requested changes will improve your model's accessibility and potential for reuse. If you have any questions or concerns about this process, please feel free to [contact us]({{ build_absolute_uri(slugurl("contact")) }}).
diff --git a/django/library/jinja2/library/review/feedback/update.jinja b/django/library/jinja2/library/review/feedback/update.jinja
index 46e746b6f..51ffdd17c 100644
--- a/django/library/jinja2/library/review/feedback/update.jinja
+++ b/django/library/jinja2/library/review/feedback/update.jinja
@@ -35,6 +35,12 @@
(click here to open in a new window/tab)
+ {% if release.is_imported %}
+
+ Please note that this model release was imported from GitHub, so the file structure may differ from
+ a typical release package in the CoMSES Model Library.
+
+ {% endif %}
You can save your progress at any time with the save button. Use the Submit button to send your finalized
diff --git a/django/library/jinja2/library/review/invitations/update.jinja b/django/library/jinja2/library/review/invitations/update.jinja
index 3e5174f17..f2aa5de99 100644
--- a/django/library/jinja2/library/review/invitations/update.jinja
+++ b/django/library/jinja2/library/review/invitations/update.jinja
@@ -29,6 +29,12 @@
If this model is not published, please do not share it with anyone else.
+ {% if release.is_imported %}
+
+ Please note that this model release was imported from GitHub, so the file structure may differ from
+ a typical release package in the CoMSES Model Library.
+
+ {% endif %}
diff --git a/django/library/management/commands/update_codebase_metadata.py b/django/library/management/commands/update_codebase_metadata.py
index 9f3f1a40d..d86bc4987 100644
--- a/django/library/management/commands/update_codebase_metadata.py
+++ b/django/library/management/commands/update_codebase_metadata.py
@@ -53,7 +53,7 @@ def update_release_packages(self):
for release in releases:
try:
fs_api = release.get_fs_api()
- fs_api.rebuild(metadata_only=True)
+ fs_api.rebuild_metadata()
except Exception as e:
errors.append((release, e))
if errors:
diff --git a/django/library/metadata.py b/django/library/metadata.py
index 535f5f46d..5a482cbcd 100644
--- a/django/library/metadata.py
+++ b/django/library/metadata.py
@@ -140,7 +140,7 @@ def _common_codebase_fields(cls, codebase) -> dict:
return dict(
type_="SoftwareSourceCode",
name=codebase.title,
- codeRepository=codebase.repository_url or None,
+ codeRepository=codebase.code_repository or None,
applicationCategory="Computational Model",
citation=[
cls.to_textual_creative_work(text)
@@ -388,19 +388,205 @@ def convert_codebase(cls, codebase, codemeta: CodeMeta | dict = None) -> DataCit
)
-def coerce_codemeta(codemeta: dict | CodeMeta, codebase=None, release=None) -> CodeMeta:
- """make sure that codemeta is a CodeMeta object. If we didn't receive anything,
- try to re-generate it from whichever object is given (codebase or release)"""
+class ReleaseMetadataConverter:
+ """Extract CoMSES CodebaseRelease metadata from various sources."""
+
+ def __init__(
+ self,
+ codemeta: CodeMeta | dict | None = None,
+ cff: CitationFileFormat | dict | None = None,
+ github_repository: dict | None = None,
+ github_release: dict | None = None,
+ ) -> dict:
+ self.codemeta = coerce_codemeta(codemeta)
+ self.cff = coerce_cff(cff)
+ self.github_repository = github_repository
+ self.github_release = github_release
+
+ def _get_field(self, source, key: str):
+ """helper to retrieve a field from either a dict or object, returning
+ None if the source either doesn't exist or doesn't contain the field"""
+ if source is None:
+ return None
+ if isinstance(source, dict):
+ return source.get(key, None)
+ return getattr(source, key, None)
+
+ def _ensure_list(self, value):
+ """return a list containing the value if it is not a list,
+ or the value if it is already a list"""
+ if isinstance(value, list):
+ return value
+ elif value is None:
+ return []
+ return [value]
+
+ def _extract_license_from_github(self) -> str | None:
+ gh_license = self._get_field(self.github_repository, "license")
+ if isinstance(gh_license, dict):
+ spdx_id = gh_license.get("spdx_id", None)
+ if spdx_id:
+ return spdx_id
+ return None
+
+ def _extract_license_from_cff(self) -> str | None:
+ cff_licenses = self._ensure_list(self._get_field(self.cff, "license"))
+ for cff_license in cff_licenses:
+ if cff_license and hasattr(cff_license, "value"):
+ return cff_license.value
+ return None
+
+ def _extract_release_notes_from_codemeta(self) -> str | None:
+ release_notes = self._get_field(self.codemeta, "releaseNotes")
+ if release_notes:
+ if isinstance(release_notes, list):
+ release_notes = release_notes[0]
+ return release_notes if isinstance(release_notes, str) else None
+ elif isinstance(release_notes, str):
+ return release_notes
+ return None
+
+ def _extract_release_notes_from_github(self) -> str | None:
+ body = self._get_field(self.github_release, "body")
+ return body if isinstance(body, str) else None
+
+ def _extract_os_from_codemeta(self) -> str:
+ codemeta_os = self._get_field(self.codemeta, "operatingSystem")
+ if not codemeta_os:
+ return ""
+ if isinstance(codemeta_os, list):
+ if not codemeta_os:
+ return ""
+ os_value = codemeta_os[0]
+ else:
+ os_value = codemeta_os
+
+ if isinstance(os_value, str):
+ os_str = os_value
+ else:
+ # try to extract a name from a structured object (e.g., {"name": "Linux"})
+ os_name = self._get_field(os_value, "name")
+ os_str = os_name if isinstance(os_name, str) else ""
+
+ if os_str:
+ normalized = os_str.lower()
+ # attempt to match the given os string to a known platform
+ if "linux" in normalized:
+ return "linux"
+ elif "mac" in normalized or "os x" in normalized:
+ return "macos"
+ elif "windows" in normalized:
+ return "windows"
+ elif "independent" in normalized or "any" in normalized:
+ return "platform_independent"
+ else:
+ return "other"
+ return ""
+
+ def _extract_programming_languages_from_codemeta(self) -> list[str] | None:
+ codemeta_langs = self._ensure_list(
+ self._get_field(self.codemeta, "programmingLanguage")
+ )
+ langs = []
+ for codemeta_lang in codemeta_langs:
+ if isinstance(codemeta_lang, str):
+ langs.append(codemeta_lang)
+ else: # try to get the name from a ComputerLanguage object
+ lang_name = self._get_field(codemeta_lang, "name")
+ if lang_name:
+ langs.append(lang_name)
+ return langs if langs else None
+
+ def _extract_programming_languages_from_github(self) -> list[str] | None:
+ gh_lang = self._get_field(self.github_repository, "language")
+ if gh_lang and isinstance(gh_lang, str):
+ return [gh_lang]
+ return None
+
+ def _extract_platforms_from_codemeta(self) -> list[str] | None:
+ runtime_platforms = self._ensure_list(
+ self._get_field(self.codemeta, "runtimePlatform")
+ )
+ platforms = []
+ for runtime_platform in runtime_platforms:
+ if isinstance(runtime_platform, str):
+ platforms.append(runtime_platform)
+ return platforms if platforms else None
+
+ def convert(self) -> dict:
+ """return a dictionary with the metadata fields for a codebase release from
+ given sources"""
+ # set priority
+ license_spdx_id = (
+ self._extract_license_from_github() or self._extract_license_from_cff()
+ )
+ release_notes = (
+ self._extract_release_notes_from_codemeta()
+ or self._extract_release_notes_from_github()
+ )
+ programming_languages = self._extract_programming_languages_from_github()
+
+ return {
+ "license_spdx_id": license_spdx_id,
+ "release_notes": release_notes,
+ "os": self._extract_os_from_codemeta(),
+ "programming_languages": programming_languages,
+ "platforms": self._extract_platforms_from_codemeta(),
+ }
+
+
+def coerce_codemeta(
+ codemeta: dict | CodeMeta | None, codebase=None, release=None
+) -> CodeMeta | None:
+ """
+ Ensure the returned value is a CodeMeta object or None.
+ - if given a CodeMeta instance, return it
+ - if given a dictionary, parse it into a CodeMeta object
+ - if given None, attempt to generate codemeta from the given codebase or release
+ - return None if the input cannot be converted or generated
+ """
if isinstance(codemeta, CodeMeta):
return codemeta
- if codebase is None:
+ if isinstance(codemeta, dict):
+ try:
+ return CodeMeta(**codemeta)
+ except Exception:
+ logger.exception("Error when parsing codemeta from dict: %s", codemeta)
+ return None
+ if codemeta is None:
if codebase:
return CodeMetaConverter.convert_codebase(codebase)
elif release:
return CodeMetaConverter.convert_release(release)
- if isinstance(codemeta, dict):
+ logger.debug(
+ "unable to coerce codemeta from inputs: codemeta=%s, codebase=%s, release=%s",
+ codemeta,
+ codebase,
+ release,
+ )
+ return None
+
+
+def coerce_cff(
+ cff: dict | CitationFileFormat | None, release=None
+) -> CitationFileFormat | None:
+ """
+ Ensure the returned value is a CitationFileFormat object or None.
+ - if given a CitationFileFormat instance, return it
+ - if given a dictionary, parse it into a CitationFileFormat object
+ - if given None, attempt to generate cff from the given release
+ - return None if the input cannot be converted or generated
+ """
+ if isinstance(cff, CitationFileFormat):
+ return cff
+ if isinstance(cff, dict):
try:
- return CodeMeta(**codemeta)
- except Exception as e:
- raise ValueError("Invalid codemeta dictionary") from e
- raise TypeError("codemeta must be a valid dictionary or CodeMeta instance")
+ return CitationFileFormat(**cff)
+ except Exception:
+ logger.exception("Error when parsing cff from dict: %s", cff)
+ return None
+ if cff is None:
+ if release:
+ return CitationFileFormatConverter.convert_release(release)
+ logger.debug("unable to coerce cff from inputs: cff=%s, release=%s", cff, release)
+ return None
diff --git a/django/library/migrations/0036_codebasegitremote_githubintegrationappinstallation_and_more.py b/django/library/migrations/0036_codebasegitremote_githubintegrationappinstallation_and_more.py
new file mode 100644
index 000000000..412959478
--- /dev/null
+++ b/django/library/migrations/0036_codebasegitremote_githubintegrationappinstallation_and_more.py
@@ -0,0 +1,670 @@
+# Generated by Django 5.2.11 on 2026-03-09 21:05
+
+import core.fields
+import django.db.models.deletion
+import modelcluster.fields
+import modelsearch.index
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("library", "0035_alter_releaselanguage_options_and_more"),
+ ("wagtailcore", "0096_referenceindex_referenceindex_source_object_and_more"),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="CodebaseGitRemote",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "is_user_repo",
+ models.BooleanField(
+ default=False, help_text="Whether this is a user-owned remote"
+ ),
+ ),
+ (
+ "is_preexisting",
+ models.BooleanField(
+ default=False,
+ help_text="Whether this remote was pre-existing or based on a codebase git mirror",
+ ),
+ ),
+ ("repo_name", models.CharField(max_length=100)),
+ (
+ "owner",
+ models.CharField(
+ help_text="Github username (or organization) who owns the remote repository",
+ max_length=100,
+ ),
+ ),
+ (
+ "url",
+ models.URLField(
+ blank=True, help_text="http URL of remote repository"
+ ),
+ ),
+ ("is_active", models.BooleanField(default=False)),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "codebase",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="git_remotes",
+ to="library.codebase",
+ ),
+ ),
+ ],
+ ),
+ migrations.CreateModel(
+ name="GithubIntegrationAppInstallation",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("github_user_id", models.BigIntegerField(unique=True)),
+ ("github_login", models.CharField(max_length=255)),
+ ("installation_id", models.BigIntegerField(unique=True)),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "user",
+ models.OneToOneField(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="github_integration_app_installation",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationConfiguration",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "is_beta",
+ models.BooleanField(
+ default=True,
+ help_text="Mark the GitHub integration feature as a beta feature",
+ ),
+ ),
+ (
+ "is_enabled_globally",
+ models.BooleanField(
+ default=False,
+ help_text="Enable GitHub integration for all users. When enabled, approved_users whitelist is ignored",
+ ),
+ ),
+ (
+ "approved_users",
+ models.ManyToManyField(
+ blank=True,
+ help_text="Whitelist of users who can access GitHub integration. Superusers/staff are implicitly included",
+ related_name="github_integration_approved",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "site",
+ models.OneToOneField(
+ editable=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="wagtailcore.site",
+ ),
+ ),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationExistingRepoInfoItem",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "icon",
+ models.CharField(
+ blank=True,
+ help_text="optional icon class (e.g., 'fas fa-link')",
+ max_length=200,
+ ),
+ ),
+ (
+ "text",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="The info item text (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "text_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_text_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="existing_repo_info_items",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationExistingRepoTutorialVideo",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "title",
+ models.CharField(
+ blank=True,
+ help_text="optional short title for the tutorial",
+ max_length=200,
+ ),
+ ),
+ (
+ "youtube_id",
+ models.CharField(
+ help_text="YouTube video id (e.g., dQw4w9WgXcQ)", max_length=32
+ ),
+ ),
+ (
+ "caption",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="optional caption (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "caption_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_caption_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="existing_repo_tutorial_videos",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationFaqEntry",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "question",
+ models.CharField(help_text="The FAQ question", max_length=500),
+ ),
+ (
+ "answer",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="The FAQ answer in Markdown format",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "answer_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_answer_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="faq_entries",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationNewRepoInfoItem",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "icon",
+ models.CharField(
+ blank=True,
+ help_text="optional icon class (e.g., 'fas fa-link')",
+ max_length=200,
+ ),
+ ),
+ (
+ "text",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="The info item text (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "text_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_text_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="new_repo_info_items",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationNewRepoTutorialVideo",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "title",
+ models.CharField(
+ blank=True,
+ help_text="optional short title for the tutorial",
+ max_length=200,
+ ),
+ ),
+ (
+ "youtube_id",
+ models.CharField(
+ help_text="YouTube video id (e.g., dQw4w9WgXcQ)", max_length=32
+ ),
+ ),
+ (
+ "caption",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="optional caption (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "caption_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_caption_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="new_repo_tutorial_videos",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitRefSyncState",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "status",
+ models.CharField(
+ choices=[
+ ("PENDING", "pending"),
+ ("RUNNING", "running"),
+ ("SUCCESS", "success"),
+ ("ERROR", "error"),
+ ],
+ default="PENDING",
+ max_length=20,
+ ),
+ ),
+ ("last_log", models.TextField(blank=True)),
+ ("full_log", models.JSONField(default=list)),
+ ("error_message", models.TextField(blank=True)),
+ ("http_status", models.IntegerField(blank=True, null=True)),
+ ("last_sync_started_at", models.DateTimeField(blank=True, null=True)),
+ ("last_sync_finished_at", models.DateTimeField(blank=True, null=True)),
+ ("batch_id", models.CharField(blank=True, max_length=36, null=True)),
+ ("tag_name", models.CharField(blank=True, max_length=128)),
+ (
+ "built_commit_sha",
+ models.CharField(
+ blank=True,
+ help_text="Last commit SHA created for this git ref in the mirror repo",
+ max_length=40,
+ ),
+ ),
+ (
+ "last_built_at",
+ models.DateTimeField(
+ blank=True,
+ help_text="Updated when the git ref is committed to the mirror repo",
+ null=True,
+ ),
+ ),
+ (
+ "pushed_commit_sha",
+ models.CharField(
+ blank=True,
+ help_text="Last commit SHA pushed to the remote repo",
+ max_length=40,
+ ),
+ ),
+ ("branch_name", models.CharField(blank=True, max_length=128)),
+ (
+ "remote",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="git_ref_sync_states",
+ to="library.codebasegitremote",
+ ),
+ ),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ migrations.AddField(
+ model_name="codebase",
+ name="main_git_ref_sync_state",
+ field=models.OneToOneField(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="codebase",
+ to="library.gitrefsyncstate",
+ ),
+ ),
+ migrations.AddField(
+ model_name="codebaserelease",
+ name="git_ref_sync_state",
+ field=models.OneToOneField(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="release",
+ to="library.gitrefsyncstate",
+ ),
+ ),
+ migrations.CreateModel(
+ name="ImportedReleaseSyncState",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "status",
+ models.CharField(
+ choices=[
+ ("PENDING", "pending"),
+ ("RUNNING", "running"),
+ ("SUCCESS", "success"),
+ ("ERROR", "error"),
+ ],
+ default="PENDING",
+ max_length=20,
+ ),
+ ),
+ ("last_log", models.TextField(blank=True)),
+ ("full_log", models.JSONField(default=list)),
+ ("error_message", models.TextField(blank=True)),
+ ("http_status", models.IntegerField(blank=True, null=True)),
+ ("last_sync_started_at", models.DateTimeField(blank=True, null=True)),
+ ("last_sync_finished_at", models.DateTimeField(blank=True, null=True)),
+ ("batch_id", models.CharField(blank=True, max_length=36, null=True)),
+ ("tag_name", models.CharField(blank=True, max_length=128)),
+ (
+ "category_manifest",
+ models.JSONField(
+ default=dict,
+ help_text="Maps file paths to categories (code, docs, data, results)",
+ ),
+ ),
+ ("github_release_id", models.CharField(max_length=128)),
+ (
+ "display_name",
+ models.CharField(
+ blank=True,
+ help_text="A human-readable name for the release, if different from the tag name",
+ max_length=255,
+ ),
+ ),
+ (
+ "html_url",
+ models.URLField(
+ blank=True,
+ help_text="URL to the (browser-viewable) page for the release on the external service",
+ ),
+ ),
+ (
+ "download_url",
+ models.URLField(
+ blank=True,
+ help_text="URL to the downloadable package for the release on the external service, ideally a zipball such as in the case of github",
+ ),
+ ),
+ (
+ "extra_data",
+ models.JSONField(
+ default=dict,
+ help_text="Additional data for the external release, ideally the full object (e.g. github release) from the external service",
+ ),
+ ),
+ (
+ "remote",
+ models.ForeignKey(
+ help_text="Remote from which this release was imported",
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="imported_sync_states",
+ to="library.codebasegitremote",
+ ),
+ ),
+ ],
+ ),
+ migrations.AddField(
+ model_name="codebaserelease",
+ name="imported_release_sync_state",
+ field=models.OneToOneField(
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ to="library.importedreleasesyncstate",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="codebasegitremote",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("is_active", True)),
+ fields=("codebase",),
+ name="single_active_remote",
+ ),
+ ),
+ migrations.AlterUniqueTogether(
+ name="codebasegitremote",
+ unique_together={("owner", "repo_name")},
+ ),
+ migrations.AlterUniqueTogether(
+ name="importedreleasesyncstate",
+ unique_together={("remote", "github_release_id")},
+ ),
+ ]
diff --git a/django/library/migrations/0037_initial_github_integration_overview_content.py b/django/library/migrations/0037_initial_github_integration_overview_content.py
new file mode 100644
index 000000000..f561de483
--- /dev/null
+++ b/django/library/migrations/0037_initial_github_integration_overview_content.py
@@ -0,0 +1,198 @@
+# Generated by Django 5.2.11 on 2026-03-09 21:06
+
+from django.db import migrations
+
+
+FAQ_ENTRIES = [
+ {
+ "question": "What GitHub permissions do you request and why?",
+ "answer": "We request read and write access to repository contents. You can either select specific repositories or allow access to all current and future repositories (though we will not attempt to access any repositories besides those connected).\r\n\r\nIf you choose the selection option, ensure that the repository you want to connect is included.",
+ },
+ {
+ "question": "Can I connect with a private GitHub repository?",
+ "answer": "No, only public GitHub repositories are supported at the moment.",
+ },
+ {
+ "question": "Can I connect with a repository owned by another user or organization?",
+ "answer": "Currently, no. We suggest forking the repository to your own account.",
+ },
+ {
+ "question": "If I have a repository connected to a model, what happens if I change the repository name, transfer ownership, make it private, or delete it?",
+ "answer": "If the repository is deleted or made private, you should disconnect the repository. If you need to change the name, you can disconnect and then reconnect with the new name. If the repository is transferred to a new owner, we suggest maintaining a fork of the repository.",
+ },
+]
+
+EXISTING_REPO_INFO_ITEMS = [
+ {
+ "icon": "fas fa-question",
+ "text": "Keep your workflow on GitHub and avoid duplicate uploading by importing tagged releases straight from GitHub",
+ },
+ {
+ "icon": "fas fa-file-archive",
+ "text": "GitHub release assets (the automatically generated zip file) are pulled into the model library and are available for download like any other release",
+ },
+ {
+ "icon": "fas fa-lock",
+ "text": "Releases imported from GitHub are private to begin with, letting you make sure everything looks right before requesting peer review or publishing",
+ },
+ {
+ "icon": "fas fa-file-alt",
+ "text": "Submissions are still required to contain source code and narrative documentation. We'll try to automatically categorize transferred files and let you make any corrections",
+ },
+ {
+ "icon": "fas fa-tags",
+ "text": "Software metadata like languages, license, etc. is extracted from the GitHub release and any existing `codemeta.json` or `CITATION.cff` files. Any missing fields can be filled in after importing",
+ },
+ {
+ "icon": "fas fa-clipboard-check",
+ "text": "Imported releases can be [peer reviewed](/reviews) just like any other submission, giving your model a DOI upon completion. Peer review **must** be requested **before** publishing.",
+ },
+ {
+ "icon": "fas fa-edit",
+ "text": "While unpublished or under review, you can still make changes to a release by updating the tag on GitHub and reimporting the release. A video tutorial demonstrates exactly how to do this.",
+ },
+]
+
+NEW_REPO_INFO_ITEMS = [
+ {
+ "icon": "fas fa-question",
+ "text": "Putting your model on GitHub makes it easier for authors and other researchers to keep track of changes and collaborate on development, while also boosting accessibility and unlocking an ecosystem of tools for making scientific software more reusable",
+ },
+ {
+ "icon": "fas fa-code-branch",
+ "text": "A git repository is automatically built for a model in the library by commiting **published** releases in order to a main branch, as well as seperate branches and tags for each release",
+ },
+ {
+ "icon": "fas fa-users",
+ "text": "In order to maintain attribution, the original submitter is set as the commit author, and all other authors as co-authors",
+ },
+ {
+ "icon": "fas fa-upload",
+ "text": "After connecting to a blank GitHub repository, you'll be able to push everything at once",
+ },
+ {
+ "icon": "fas fa-sync-alt",
+ "text": "New releases published in the model library or metadata changes to existing releases will be automatically added to the git repository but will require input to push to GitHub",
+ },
+ {
+ "icon": "fas fa-download",
+ "text": "We'll never overwrite any changes you make to any branch on the GitHub repository and if you make any new releases there, you'll be able to import them back into the model library",
+ },
+]
+
+EXISTING_REPO_TUTORIAL_VIDEOS = [
+ {
+ "title": "",
+ "youtube_id": "-",
+ "caption": "How to set up an existing repository and import releases into the model library",
+ },
+ {
+ "title": "",
+ "youtube_id": "-",
+ "caption": "How to update a GitHub release and re-import it",
+ },
+]
+
+NEW_REPO_TUTORIAL_VIDEOS = [
+ {
+ "title": "",
+ "youtube_id": "-",
+ "caption": "How to set up and push a new git repository for your model in the library",
+ }
+]
+
+
+def _get_github_config_for_default_site(apps):
+ Site = apps.get_model("wagtailcore", "Site")
+ GitHubIntegrationConfiguration = apps.get_model(
+ "library", "GitHubIntegrationConfiguration"
+ )
+ default_site = Site.objects.get(is_default_site=True)
+ config, _ = GitHubIntegrationConfiguration.objects.get_or_create(site=default_site)
+ return config
+
+
+def delete_existing_content(apps, schema_editor):
+ config = _get_github_config_for_default_site(apps)
+ apps.get_model("library", "GitHubIntegrationFaqEntry").objects.filter(
+ configuration=config
+ ).delete()
+ apps.get_model("library", "GitHubIntegrationExistingRepoInfoItem").objects.filter(
+ configuration=config
+ ).delete()
+ apps.get_model("library", "GitHubIntegrationNewRepoInfoItem").objects.filter(
+ configuration=config
+ ).delete()
+ apps.get_model(
+ "library", "GitHubIntegrationExistingRepoTutorialVideo"
+ ).objects.filter(configuration=config).delete()
+ apps.get_model("library", "GitHubIntegrationNewRepoTutorialVideo").objects.filter(
+ configuration=config
+ ).delete()
+
+
+def populate_initial_content(apps, schema_editor):
+ config = _get_github_config_for_default_site(apps)
+
+ GitHubIntegrationFaqEntry = apps.get_model("library", "GitHubIntegrationFaqEntry")
+ GitHubIntegrationExistingRepoInfoItem = apps.get_model(
+ "library", "GitHubIntegrationExistingRepoInfoItem"
+ )
+ GitHubIntegrationNewRepoInfoItem = apps.get_model(
+ "library", "GitHubIntegrationNewRepoInfoItem"
+ )
+ GitHubIntegrationExistingRepoTutorialVideo = apps.get_model(
+ "library", "GitHubIntegrationExistingRepoTutorialVideo"
+ )
+ GitHubIntegrationNewRepoTutorialVideo = apps.get_model(
+ "library", "GitHubIntegrationNewRepoTutorialVideo"
+ )
+
+ for sort_order, row in enumerate(FAQ_ENTRIES):
+ GitHubIntegrationFaqEntry.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ question=row["question"],
+ answer=row["answer"],
+ )
+ for sort_order, row in enumerate(EXISTING_REPO_INFO_ITEMS):
+ GitHubIntegrationExistingRepoInfoItem.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ icon=row["icon"],
+ text=row["text"],
+ )
+ for sort_order, row in enumerate(NEW_REPO_INFO_ITEMS):
+ GitHubIntegrationNewRepoInfoItem.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ icon=row["icon"],
+ text=row["text"],
+ )
+ for sort_order, row in enumerate(EXISTING_REPO_TUTORIAL_VIDEOS):
+ GitHubIntegrationExistingRepoTutorialVideo.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ title=row["title"],
+ youtube_id=row["youtube_id"],
+ caption=row["caption"],
+ )
+ for sort_order, row in enumerate(NEW_REPO_TUTORIAL_VIDEOS):
+ GitHubIntegrationNewRepoTutorialVideo.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ title=row["title"],
+ youtube_id=row["youtube_id"],
+ caption=row["caption"],
+ )
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("library", "0036_codebasegitremote_githubintegrationappinstallation_and_more"),
+ ]
+
+ operations = [
+ migrations.RunPython(populate_initial_content, delete_existing_content),
+ ]
diff --git a/django/library/models.py b/django/library/models.py
index e91846749..500936412 100644
--- a/django/library/models.py
+++ b/django/library/models.py
@@ -1,4 +1,5 @@
import hashlib
+import io
import json
import yaml
import logging
@@ -13,11 +14,13 @@
from collections import defaultdict
from datetime import date, timedelta
+from allauth.socialaccount.models import SocialAccount
from django.conf import settings
+from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.postgres.fields import ArrayField
from django.core.files.images import ImageFile
from django.core.files.storage import FileSystemStorage
-from django.db import models, transaction
+from django.db import IntegrityError, models, transaction
from django.db.models import Prefetch, Q, Count, Max
from django.utils.functional import cached_property
from django.urls import reverse
@@ -30,7 +33,7 @@
from modelcluster.models import ClusterableModel
from rest_framework.exceptions import ValidationError, UnsupportedMediaType
from taggit.models import TaggedItemBase
-from wagtail.admin.panels import FieldPanel
+from wagtail.admin.panels import FieldPanel, InlinePanel, HelpPanel
from wagtail.coreutils import string_to_ascii
from wagtail.images.models import (
Image,
@@ -40,8 +43,11 @@
get_upload_to,
ImageQuerySet,
)
+from wagtail.models import Orderable
from wagtail.search import index
from wagtail.snippets.models import register_snippet
+from wagtail.contrib.settings.models import BaseSiteSetting, register_setting
+from django.utils.html import format_html
from core import fs
from core.backends import add_to_comses_permission_whitelist
@@ -52,8 +58,9 @@
from core.view_helpers import get_search_queryset
from .fs import (
CodebaseReleaseFsApi,
+ ImportedCodebaseReleaseFsApi,
StagingDirectories,
- FileCategoryDirectories,
+ FileCategories,
MessageLevels,
)
from .metadata import CodeMetaConverter, DataCiteConverter, CitationFileFormatConverter
@@ -616,6 +623,427 @@ def updated_after(self, start_date, end_date=None, **kwargs):
return new_codebases, updated_codebases, releases
+class GithubIntegrationAppInstallation(models.Model):
+ """Represents a user's installation of the Github Integration App"""
+
+ user = models.OneToOneField(
+ settings.AUTH_USER_MODEL,
+ related_name="github_integration_app_installation",
+ on_delete=models.CASCADE,
+ )
+ github_user_id = models.BigIntegerField(unique=True)
+ github_login = models.CharField(max_length=255)
+ installation_id = models.BigIntegerField(unique=True)
+ date_created = models.DateTimeField(auto_now_add=True)
+ last_modified = models.DateTimeField(auto_now=True)
+
+ def __str__(self):
+ return f"{self.github_login} (installation_id={self.installation_id})"
+
+
+class CodebaseGitRemote(models.Model):
+ """Represents a remote repository associated with a `Codebase` that can be pushed to or imported from"""
+
+ is_user_repo = models.BooleanField(
+ default=False, help_text="Whether this is a user-owned remote"
+ )
+ is_preexisting = models.BooleanField(
+ default=False,
+ help_text="Whether this remote was pre-existing or based on a codebase git mirror",
+ )
+ repo_name = models.CharField(max_length=100)
+ owner = models.CharField(
+ max_length=100,
+ help_text="Github username (or organization) who owns the remote repository",
+ )
+ url = models.URLField(
+ blank=True,
+ help_text=_("http URL of remote repository"),
+ )
+ is_active = models.BooleanField(default=False)
+ codebase = models.ForeignKey(
+ "Codebase",
+ related_name="git_remotes",
+ on_delete=models.CASCADE,
+ )
+ date_created = models.DateTimeField(auto_now_add=True)
+ last_modified = models.DateTimeField(auto_now=True)
+
+ @property
+ def installation(self) -> GithubIntegrationAppInstallation | None:
+ if self.is_user_repo:
+ return self.codebase.submitter.github_integration_app_installation
+ return None
+
+ def __str__(self):
+ return f"{self.owner}/{self.repo_name} (active={self.is_active}))"
+
+ class Meta:
+ unique_together = ("owner", "repo_name")
+ constraints = [
+ models.UniqueConstraint(
+ fields=["codebase"],
+ condition=models.Q(is_active=True),
+ name="single_active_remote",
+ ),
+ ]
+
+
+class BaseReleaseSyncState(models.Model):
+ class Status(models.TextChoices):
+ PENDING = "PENDING", _("pending")
+ RUNNING = "RUNNING", _("running")
+ SUCCESS = "SUCCESS", _("success")
+ ERROR = "ERROR", _("error")
+
+ date_created = models.DateTimeField(auto_now_add=True)
+ last_modified = models.DateTimeField(auto_now=True)
+ status = models.CharField(
+ max_length=20, choices=Status.choices, default=Status.PENDING
+ )
+ last_log = models.TextField(blank=True)
+ full_log = models.JSONField(default=list)
+ error_message = models.TextField(blank=True)
+ http_status = models.IntegerField(null=True, blank=True)
+ last_sync_started_at = models.DateTimeField(null=True, blank=True)
+ last_sync_finished_at = models.DateTimeField(null=True, blank=True)
+ batch_id = models.CharField(max_length=36, blank=True, null=True)
+ tag_name = models.CharField(max_length=128, blank=True)
+
+ def __str__(self):
+ return f"status: {self.status}, last_log: {self.last_log}, error_message: {self.error_message}"
+
+ def _append_log(self, message: str, now):
+ logs = list(self.full_log or [])
+ logs.append({"t": now.isoformat(), "msg": message})
+ self.full_log = logs
+ self.last_log = message
+
+ def mark_running(self):
+ """set status to RUNNING and stamp last_sync_started_at"""
+ self.status = self.Status.RUNNING
+ self.last_sync_started_at = timezone.now()
+ self.save(update_fields=["status", "last_sync_started_at", "last_modified"])
+ return self
+
+ def log_failure(self, message: str):
+ """mark this sync state as ERROR and append a log entry"""
+ now = timezone.now()
+ self.status = self.Status.ERROR
+ self.error_message = message
+ self.last_sync_finished_at = now
+ self._append_log(message, now)
+ self.save()
+ return self
+
+ def log_success(self, message: str | None = None):
+ """mark this sync state as SUCCESS and optionally append a log entry"""
+ now = timezone.now()
+ self.status = self.Status.SUCCESS
+ self.last_sync_finished_at = now
+ if message:
+ self._append_log(message, now)
+ self.save()
+ return self
+
+ class Meta:
+ abstract = True
+
+
+class GitRefSyncState(BaseReleaseSyncState):
+ """
+ Represents the sync state (to a remote) of a git ref (release branch/tag or the codebase main branch)
+ that was built from a submission
+
+ exactly one is attached to either a CodebaseRelease (release branch / tag)
+ or a Codebase (main branch)
+
+ created with remote=None and assigned to a remote once a push is attempted, when swapping remotes,
+ the state is reset and assigned to the new remote
+ """
+
+ remote = models.ForeignKey(
+ "library.CodebaseGitRemote",
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ related_name="git_ref_sync_states",
+ )
+ built_commit_sha = models.CharField(
+ max_length=40,
+ blank=True,
+ help_text=_("Last commit SHA created for this git ref in the mirror repo"),
+ )
+ last_built_at = models.DateTimeField(
+ null=True,
+ blank=True,
+ help_text=_("Updated when the git ref is committed to the mirror repo"),
+ )
+ pushed_commit_sha = models.CharField(
+ max_length=40,
+ blank=True,
+ help_text=_("Last commit SHA pushed to the remote repo"),
+ )
+ branch_name = models.CharField(max_length=128, blank=True)
+
+ @classmethod
+ def latest_release_ref_for_codebase(cls, codebase):
+ return (
+ cls.objects.filter(release__codebase=codebase)
+ .select_related("release")
+ .order_by("release__version_number")
+ .last()
+ )
+
+ @classmethod
+ def releases_without_git_ref_sync_state(cls, codebase):
+ built_ids = cls.objects.filter(release__codebase=codebase).values_list(
+ "release__id", flat=True
+ )
+ return codebase.releases.internal().public().exclude(id__in=built_ids)
+
+ @classmethod
+ def for_release(cls, release):
+ """Fetch or create the release's git ref sync state."""
+ state = getattr(release, "git_ref_sync_state", None)
+ if state:
+ return state
+ state = cls.objects.create(status=cls.Status.PENDING)
+ release.git_ref_sync_state = state
+ release.save(
+ rebuild_metadata=False,
+ update_fields=["git_ref_sync_state", "last_modified"],
+ )
+ return state
+
+ @classmethod
+ def for_codebase(cls, codebase, branch_name="main"):
+ """Fetch or create the codebase main branch git ref sync state."""
+ state = getattr(codebase, "main_git_ref_sync_state", None)
+ if state:
+ return state
+ state = cls.objects.create(status=cls.Status.PENDING, branch_name=branch_name)
+ codebase.main_git_ref_sync_state = state
+ codebase.save(
+ rebuild_metadata=False,
+ rebuild_release_metadata=False,
+ update_fields=["main_git_ref_sync_state", "last_modified"],
+ )
+ return state
+
+ @classmethod
+ def get_last_remote_for_codebase(cls, codebase):
+ """return the remote that git ref sync states are currently bound to, if any.
+
+ checks the main branch state first, then falls back to the first release state.
+ returns None if no states have a remote assigned.
+ """
+ main_state = getattr(codebase, "main_git_ref_sync_state", None)
+ if main_state and main_state.remote:
+ return main_state.remote
+ release_state = cls.objects.filter(
+ release__codebase=codebase, remote__isnull=False
+ ).first()
+ if release_state:
+ return release_state.remote
+ return None
+
+ @classmethod
+ def reassign_remotes_for_codebase(cls, codebase, remote):
+ """reassign all git ref sync states for a codebase to a new remote and
+ DESTROY push state for the prior remote"""
+ qs = cls.objects.filter(release__codebase=codebase)
+ for state in qs:
+ state.reassign_remote(remote)
+ main_state = getattr(codebase, "main_git_ref_sync_state", None)
+ if main_state:
+ main_state.reassign_remote(remote)
+
+ @classmethod
+ def start_all_push_jobs_for_codebase(cls, codebase, remote) -> bool:
+ """set remotes and mark pushable refs (main + releases) as RUNNING"""
+ started = False
+ # start main branch job if needed
+ main_state = codebase.get_or_create_main_git_ref_sync_state()
+ main_state.set_remote(remote)
+ if main_state.can_push():
+ main_state.mark_running()
+ started = True
+ # start release jobs that can be pushed
+ releases = codebase.ordered_releases_list(internal_only=True, asc=True)
+ for release in releases:
+ state = release.get_or_create_git_ref_sync_state()
+ state.set_remote(remote)
+ if state.can_push():
+ state.mark_running()
+ started = True
+ return started
+
+ def reassign_remote(self, remote):
+ """bind to a new remote and reset push status/logs"""
+ self.remote = remote
+ self.pushed_commit_sha = ""
+ self.status = self.Status.PENDING
+ self.last_log = ""
+ self.error_message = ""
+ self.http_status = None
+ self.last_sync_started_at = None
+ self.last_sync_finished_at = None
+ # keeping this for now, might be useful if something goes wrong
+ # self.full_log = []
+ self.save()
+ return self
+
+ def record_build(self, commit_sha, tag_name=None, branch_name=None):
+ """record the latest commit hash and optionally tag/branch names if given and not already set"""
+ self.built_commit_sha = commit_sha
+ # only update if given and not already set
+ if tag_name and not (self.tag_name or "").strip():
+ self.tag_name = tag_name
+ if branch_name and not (self.branch_name or "").strip():
+ self.branch_name = branch_name
+ self.last_built_at = timezone.now()
+ self.save()
+ return self
+
+ def set_remote(self, remote):
+ """set the remote, only if not already set to something different"""
+ if self.remote and self.remote.id != remote.id:
+ raise ValueError("git ref sync state already bound to a different remote")
+ self.remote = remote
+ self.save()
+ return self
+
+ def record_push_success(self, commit_sha, summary=None):
+ self.pushed_commit_sha = commit_sha
+ self.save(update_fields=["pushed_commit_sha", "last_modified"])
+ return self.log_success(summary)
+
+ def can_push(self) -> bool:
+ """
+ return true when a git ref is pushable, either:
+ - push failed
+ - no commit sha has been pushed
+ - or the pushed sha is out of date
+ """
+ if self.status == self.Status.ERROR:
+ return True
+ if not self.pushed_commit_sha:
+ return True
+ if self.built_commit_sha and self.built_commit_sha != self.pushed_commit_sha:
+ return True
+ return False
+
+
+class ImportedReleaseSyncState(BaseReleaseSyncState):
+ """Represents the state of the process involved in importing a release from GitHub.
+ Will only exist for releases that have at least been attempted to be imported from GitHub
+
+ only one will exist for a given CodebaseRelease, since a release can only be imported once.
+ though multiple can exist for a GitHub release, only if they are from different remotes.
+ """
+
+ # remote is the source repository this GitHub release came from
+ remote = models.ForeignKey(
+ "library.CodebaseGitRemote",
+ null=True,
+ on_delete=models.SET_NULL,
+ related_name="imported_sync_states",
+ help_text=_("Remote from which this release was imported"),
+ )
+ category_manifest = models.JSONField(
+ default=dict,
+ help_text="Maps file paths to categories (code, docs, data, results)",
+ )
+ github_release_id = models.CharField(
+ max_length=128,
+ )
+ display_name = models.CharField(
+ max_length=255,
+ blank=True,
+ help_text="A human-readable name for the release, if different from the tag name",
+ )
+ html_url = models.URLField(
+ blank=True,
+ help_text="URL to the (browser-viewable) page for the release on the external service",
+ )
+ download_url = models.URLField(
+ blank=True,
+ help_text="URL to the downloadable package for the release on the external service, ideally a zipball such as in the case of github",
+ )
+ extra_data = models.JSONField(
+ default=dict,
+ help_text="Additional data for the external release, ideally the full object (e.g. github release) from the external service",
+ )
+
+ class Meta:
+ unique_together = ("remote", "github_release_id")
+
+ # manager-style helpers
+ @classmethod
+ def for_github_release(cls, remote, gh_release_raw: dict):
+ """create or update a sync state from a github release object from the github API.
+ only updates when tracked fields change
+ """
+ release_id = str(gh_release_raw.get("id", ""))
+ if not release_id:
+ raise ValueError("missing github release id")
+ tag_name = gh_release_raw.get("tag_name")
+ display_name = gh_release_raw.get("name")
+ html_url = gh_release_raw.get("html_url")
+ download_url = gh_release_raw.get("zipball_url")
+
+ obj, _ = cls.objects.get_or_create(
+ remote=remote,
+ github_release_id=release_id,
+ defaults={
+ "tag_name": tag_name,
+ "display_name": display_name,
+ "html_url": html_url,
+ "download_url": download_url,
+ "extra_data": gh_release_raw,
+ "status": cls.Status.PENDING,
+ },
+ )
+ changed = False
+ if obj.tag_name != tag_name:
+ obj.tag_name = tag_name
+ changed = True
+ if obj.display_name != display_name:
+ obj.display_name = display_name
+ changed = True
+ if obj.html_url != html_url:
+ obj.html_url = html_url
+ changed = True
+ if obj.download_url != download_url:
+ obj.download_url = download_url
+ changed = True
+ # store full release object
+ if (obj.extra_data or {}) != (gh_release_raw or {}):
+ obj.extra_data = gh_release_raw or {}
+ changed = True
+
+ if changed:
+ obj.save()
+ return obj
+
+ def can_reimport(self) -> bool:
+ """
+ return true when a re-import should be allowed:
+ - job has completed (SUCCESS) or failed (ERROR)
+ - and either no CodebaseRelease exists yet for this sync state
+ - or the existing CodebaseRelease is not live (i.e., not published)
+ """
+ if self.status not in [self.Status.SUCCESS, self.Status.ERROR]:
+ return False
+ release = CodebaseRelease.objects.filter(
+ imported_release_sync_state=self
+ ).first()
+ if not release:
+ return True
+ return not release.live
+
+
@add_to_comses_permission_whitelist
class Codebase(index.Indexed, ModeratedContent, ClusterableModel):
"""
@@ -699,6 +1127,13 @@ class Codebase(index.Indexed, ModeratedContent, ClusterableModel):
"JSON metadata conforming to the codemeta schema. Cached as of the last update"
),
)
+ main_git_ref_sync_state = models.OneToOneField(
+ "library.GitRefSyncState",
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ related_name="codebase",
+ )
objects = CodebaseQuerySet.as_manager()
@@ -747,6 +1182,13 @@ def datacite_temp(self):
self, codemeta=self.codemeta_snapshot or None
)
+ @property
+ def code_repository(self):
+ git_remote = self.active_git_remote
+ if self.pk and git_remote:
+ return git_remote.url
+ return self.repository_url
+
@property
def is_replication(self):
return bool(self.replication_text.strip())
@@ -812,6 +1254,10 @@ def media_dir(self, *args):
def relative_media_path(self, *args):
return pathlib.Path(str(self.uuid), "media", *args)
+ @property
+ def active_git_remote(self):
+ return self.git_remotes.filter(is_active=True).first()
+
@property
def summarized_description(self):
if self.summary:
@@ -833,6 +1279,38 @@ def base_library_dir(self):
def base_git_dir(self):
return pathlib.Path(settings.REPOSITORY_ROOT, str(self.uuid))
+ def get_or_create_main_git_ref_sync_state(self, branch_name="main"):
+ """return the git ref sync state for this codebase's main branch"""
+ return GitRefSyncState.for_codebase(self, branch_name)
+
+ def create_remote(self, owner, repo_name, **kwargs):
+ """create a new remote for this codebase, or get an existing one if there is a
+ matching remote that was previously deactivated"""
+ existing_remote = CodebaseGitRemote.objects.filter(
+ codebase=self,
+ owner=owner,
+ repo_name=repo_name,
+ ).first()
+ if existing_remote:
+ existing_remote.is_active = True
+ existing_remote.save(update_fields=["is_active", "last_modified"])
+ return existing_remote
+ remote = CodebaseGitRemote.objects.create(
+ codebase=self,
+ owner=owner,
+ repo_name=repo_name,
+ **kwargs,
+ )
+ return remote
+
+ def latest_release_git_ref_sync_state(self):
+ """return the latest (by version number) git ref sync state for this codebase"""
+ return GitRefSyncState.latest_release_ref_for_codebase(self)
+
+ def releases_without_git_ref_sync_state(self):
+ """return internal (non-github imported), public releases that have not yet been added to the git repo"""
+ return GitRefSyncState.releases_without_git_ref_sync_state(self)
+
@property
def publication_year(self):
return (
@@ -873,6 +1351,8 @@ def _get_unique_contributors(self, release_contributors):
"""
Returns a unique, unordered set of Contributor objects given an initial filtered set of ReleaseContributors for this Codebase.
"""
+ if not self.pk:
+ return Contributor.objects.none()
return Contributor.objects.filter(
id__in=release_contributors.for_codebase(self).values("contributor_id")
)
@@ -903,21 +1383,23 @@ def download_count(self):
release__codebase__id=self.id
).count()
- def ordered_releases_list(self, has_change_perm=False, asc=True, **kwargs):
+ def ordered_releases_list(
+ self, has_change_perm=False, asc=True, internal_only=False, **kwargs
+ ):
"""
- list public releases (or all release if has_change_perm is True) in ascending (default) or descending order
+ list public releases (or all release if has_change_perm is True) in ascending (default) or descending order.
+ If internal_only is True, only return internal (directly uploaded) releases
"""
if has_change_perm:
releases = self.releases.filter(**kwargs)
else:
- releases = self.public_releases(**kwargs)
+ releases = self.releases.public(**kwargs)
+ if internal_only:
+ releases = releases.internal()
releases_list = list(releases)
releases_list.sort(key=lambda r: Version(r.version_number), reverse=not asc)
return releases_list
- def public_releases(self, **kwargs):
- return self.releases.filter(status=CodebaseRelease.Status.PUBLISHED, **kwargs)
-
@classmethod
def get_list_url(cls):
return reverse("library:codebase-list")
@@ -1030,14 +1512,20 @@ def import_media(self, fileobj, user=None, title=None, images_only=True):
return image_metadata
@transaction.atomic
- def get_or_create_draft(self):
+ def get_or_create_draft(self, initial_version: str | None = None):
existing_draft = self.releases.filter(
status=CodebaseRelease.Status.DRAFT
).first()
if existing_draft:
return existing_draft
- draft_release = self.create_release()
+ # allow for overriding the version number of the initial draft
+ # this is currently used to create a less conflict-prone 0.0.1 draft when
+ # someone wants to import all their work with the github integration
+ if initial_version:
+ draft_release = self.create_release(version_number=initial_version)
+ else:
+ draft_release = self.create_release()
# reset fields that should not be copied over to a new draft
draft_release.doi = None
draft_release.release_notes = ""
@@ -1047,8 +1535,11 @@ def get_or_create_draft(self):
@transaction.atomic
def create_review_draft_from_release(self, source_release):
- # create a new "under review" release from an existing release
- # (should only be called from a publishable release)
+ """create a new "under review" release from an existing release
+
+ (should only be called from a publishable, non-external release)
+ """
+
source_id = source_release.id
review_draft = self.create_release(
status=CodebaseRelease.Status.UNDER_REVIEW, source_release=source_release
@@ -1065,6 +1556,8 @@ def create_release_from_source(self, source_release, release_metadata):
# set source_release.id to None to create a new release
# see https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances
source_release.id = None
+ source_release.imported_release_sync_state = None
+ source_release.git_ref_sync_state = None
source_release._state.adding = True
source_release.__dict__.update(**release_metadata)
source_release.save()
@@ -1123,7 +1616,7 @@ def create_release(
def save(self, rebuild_metadata=True, rebuild_release_metadata=True, **kwargs):
"""save the codebase and optionally rebuild metadata by updating codemeta_snapshot.
- If rebuild_release_metadata is True, all releases will be saved to trigger metadata updates
+ If rebuild_release_metadata is True, all internal releases will be saved to trigger metadata updates
"""
if rebuild_metadata and self.pk:
logger.debug("Building codemeta for codebase: %s", self)
@@ -1132,7 +1625,7 @@ def save(self, rebuild_metadata=True, rebuild_release_metadata=True, **kwargs):
# saving releases will trigger metadata rebuilding and updating
# the fs and git mirror if one exists
if rebuild_metadata and rebuild_release_metadata:
- for release in self.releases.all():
+ for release in self.releases.internal():
release.save(rebuild_metadata=True)
@classmethod
@@ -1231,6 +1724,14 @@ def with_codebase(self):
def with_submitter(self):
return self.prefetch_related("submitter")
+ def internal(self, **kwargs):
+ """returns only releases that are locally uploaded and not archived from an external service"""
+ return self.filter(imported_release_sync_state__isnull=True, **kwargs)
+
+ def imported(self, **kwargs):
+ """returns only releases that are imported from an external service such as GitHub"""
+ return self.filter(imported_release_sync_state__isnull=False, **kwargs)
+
def public(self, **kwargs):
return self.filter(status=CodebaseRelease.Status.PUBLISHED, **kwargs)
@@ -1320,6 +1821,19 @@ class Status(models.TextChoices):
max_length=32,
)
+ imported_release_sync_state = models.OneToOneField(
+ ImportedReleaseSyncState,
+ null=True,
+ on_delete=models.SET_NULL,
+ )
+ git_ref_sync_state = models.OneToOneField(
+ "library.GitRefSyncState",
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ related_name="release",
+ )
+
first_published_at = models.DateTimeField(null=True, blank=True)
last_published_on = models.DateTimeField(null=True, blank=True)
@@ -1496,6 +2010,10 @@ def get_notify_reviewers_of_changes_url(self):
},
)
+ def get_or_create_git_ref_sync_state(self):
+ """Return the git ref sync state for this release (creating if needed)."""
+ return GitRefSyncState.for_release(self)
+
def get_review(self):
review = getattr(self, "review", None)
return review if review and not review.closed else None
@@ -1563,27 +2081,21 @@ def validate_metadata(self):
return True
def validate_uploaded_files(self):
+ """Validates that this CodebaseRelease has the required files to be publish()-able
+ - at least one code file
+ - at least one documentation file
+ """
fs_api = self.get_fs_api()
- storage = fs_api.get_stage_storage(StagingDirectories.sip)
- code_msg = self.check_files(storage, FileCategoryDirectories.code)
- docs_msg = self.check_files(storage, FileCategoryDirectories.docs)
- msg = " ".join(m for m in [code_msg, docs_msg] if m)
- if msg:
- raise ValidationError(msg)
+ missing_categories = []
+ if not fs_api.check_category_file_exists(FileCategories.code):
+ missing_categories.append(FileCategories.code.name)
+ if not fs_api.check_category_file_exists(FileCategories.docs):
+ missing_categories.append(FileCategories.docs.name)
+ if missing_categories:
+ missing_str = " and ".join(missing_categories)
+ raise ValidationError(f"Must have at least one {missing_str} file.")
return True
- @staticmethod
- def check_files(storage, category: FileCategoryDirectories):
- # FIXME: document and/or refactor this API, should raise an exception or ...
- # currently returns an error message or ""
- uploaded_files = []
- if storage.exists(category.name):
- uploaded_files = list(storage.list(category))
- if uploaded_files:
- return ""
- else:
- return f"Must have at least one {category.name} file."
-
@property
def version_info(self):
if self.version_number:
@@ -1619,9 +2131,13 @@ def is_peer_review_requestable(self):
1. ready to be published
2. has not already been peer reviewed
3. a related PeerReview does not exist
+ 4. is not an imported, published release
"""
return (
- self.is_publishable and not self.peer_reviewed and self.get_review() is None
+ self.is_publishable
+ and not self.peer_reviewed
+ and self.get_review() is None
+ and not (self.is_imported and self.is_published)
)
@property
@@ -1858,12 +2374,19 @@ def get_status_color(self):
}
return COLOR_MAP.get(self.status)
+ @property
+ def is_imported(self):
+ return self.imported_release_sync_state is not None
+
def create_or_update_codemeta(self, force=True):
return self.get_fs_api().create_or_update_codemeta(force=force)
def get_fs_api(
self, mimetype_mismatch_message_level=MessageLevels.error
- ) -> CodebaseReleaseFsApi:
+ ) -> CodebaseReleaseFsApi | ImportedCodebaseReleaseFsApi:
+ """Factory method to return the appropriate filesystem API for this release."""
+ if self.is_imported:
+ return ImportedCodebaseReleaseFsApi.initialize(self)
return CodebaseReleaseFsApi.initialize(
self, mimetype_mismatch_message_level=mimetype_mismatch_message_level
)
@@ -1911,6 +2434,13 @@ def publish(self):
schedule_mint_public_doi(
self.id, dry_run=settings.DEPLOY_ENVIRONMENT.is_development
)
+ if GitRefSyncState.objects.filter(release__codebase=self.codebase).exists():
+ # if this codebase has a local git repo built by us, append the new release to it
+ from .tasks import append_releases_to_local_git_repo
+
+ transaction.on_commit(
+ lambda: append_releases_to_local_git_repo(self.codebase.id)
+ )
def _publish(self):
if not self.live:
@@ -1995,18 +2525,25 @@ def save(self, rebuild_metadata=True, defer_fs=True, **kwargs):
if not rebuild_metadata:
super().save(**kwargs)
else:
- logger.debug("Building codemeta for release: %s", self)
- old_codemeta = self.codemeta_snapshot
- self.codemeta_snapshot = self.codemeta.dict(serialize=True)
- super().save(**kwargs)
-
- if old_codemeta != self.codemeta_snapshot:
- if defer_fs:
- from .tasks import update_fs_release_metadata
-
- update_fs_release_metadata(self.id)
- else:
- self.get_fs_api().rebuild(metadata_only=True)
+ if not self.pk:
+ # save first so m2m/reverse relations used by codemeta generation can be accessed
+ # do not mess with the filesystem if this is a new release
+ super().save(**kwargs)
+ self.codemeta_snapshot = self.codemeta.dict(serialize=True)
+ super().save(update_fields=["codemeta_snapshot"])
+ else:
+ logger.debug("Building codemeta for release: %s", self)
+ old_codemeta = self.codemeta_snapshot
+ self.codemeta_snapshot = self.codemeta.dict(serialize=True)
+ super().save(**kwargs)
+
+ if old_codemeta != self.codemeta_snapshot:
+ if defer_fs:
+ from .tasks import update_fs_release_metadata
+
+ update_fs_release_metadata(self.id)
+ else:
+ self.get_fs_api().rebuild_metadata()
@classmethod
def get_indexed_objects(cls):
@@ -2778,9 +3315,16 @@ def editor_called_for_revisions(self, editor: MemberProfile):
review.status = ReviewStatus.AWAITING_AUTHOR_CHANGES
review.save()
recipients = {review.submitter.email, review.codebase_release.submitter.email}
+
+ if review.codebase_release.is_imported:
+ template_name = (
+ "library/review/email/model_revisions_requested_imported.jinja"
+ )
+ else:
+ template_name = "library/review/email/model_revisions_requested.jinja"
send_markdown_email(
subject="Peer review: revisions requested",
- template_name="library/review/email/model_revisions_requested.jinja",
+ template_name=template_name,
context={"review": review, "feedback": self},
to=recipients,
bcc=[settings.REVIEW_EDITOR_EMAIL],
@@ -2862,9 +3406,7 @@ def __init__(self, release: CodebaseRelease):
# raise ValueError("Invalid release with no License")
# FIXME: set all of these fields explicitly
- self.code_repository = (
- codebase.repository_url or "https://github.com/comses-model-library/"
- )
+ self.code_repository = codebase.code_repository
self.permanent_url = release.permanent_url
def convert_keywords(self):
@@ -3347,3 +3889,179 @@ def __str__(self):
HTTP Status: {self.http_status}, Message: {self.message},
Hash: {self.metadata_hash}, DOI: {self.doi}
"""
+
+
+class GitHubIntegrationFaqEntry(index.Indexed, Orderable):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="faq_entries",
+ on_delete=models.CASCADE,
+ )
+ question = models.CharField(max_length=500, help_text=_("The FAQ question"))
+ answer = MarkdownField(help_text=_("The FAQ answer in Markdown format"))
+
+ class Meta:
+ ordering = ["sort_order", "id"]
+
+
+class GitHubIntegrationInfoItemBase(index.Indexed, Orderable):
+ icon = models.CharField(
+ max_length=200,
+ help_text=_("optional icon class (e.g., 'fas fa-link')"),
+ blank=True,
+ )
+ text = MarkdownField(help_text=_("The info item text (Markdown)"))
+
+ panels = [
+ FieldPanel("icon"),
+ FieldPanel("text"),
+ ]
+
+ class Meta:
+ abstract = True
+ ordering = ["sort_order", "id"]
+
+
+class GitHubIntegrationExistingRepoInfoItem(GitHubIntegrationInfoItemBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="existing_repo_info_items",
+ on_delete=models.CASCADE,
+ )
+
+
+class GitHubIntegrationNewRepoInfoItem(GitHubIntegrationInfoItemBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="new_repo_info_items",
+ on_delete=models.CASCADE,
+ )
+
+
+class GitHubIntegrationTutorialVideoBase(index.Indexed, Orderable):
+ title = models.CharField(
+ blank=True, max_length=200, help_text=_("optional short title for the tutorial")
+ )
+ youtube_id = models.CharField(
+ max_length=32,
+ help_text=_("YouTube video id (e.g., dQw4w9WgXcQ)"),
+ )
+ caption = MarkdownField(blank=True, help_text=_("optional caption (Markdown)"))
+
+ panels = [
+ FieldPanel("title"),
+ FieldPanel("youtube_id"),
+ FieldPanel("caption"),
+ ]
+
+ class Meta:
+ abstract = True
+ ordering = ["sort_order", "id"]
+
+ def __str__(self):
+ if self.title:
+ return f"{self.title} ({self.youtube_id})"
+ return self.youtube_id
+
+
+class GitHubIntegrationExistingRepoTutorialVideo(GitHubIntegrationTutorialVideoBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="existing_repo_tutorial_videos",
+ on_delete=models.CASCADE,
+ )
+
+
+class GitHubIntegrationNewRepoTutorialVideo(GitHubIntegrationTutorialVideoBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="new_repo_tutorial_videos",
+ on_delete=models.CASCADE,
+ )
+
+
+@register_setting
+class GitHubIntegrationConfiguration(BaseSiteSetting, ClusterableModel):
+ is_beta = models.BooleanField(
+ default=True,
+ help_text=_("Mark the GitHub integration feature as a beta feature"),
+ )
+ is_enabled_globally = models.BooleanField(
+ default=False,
+ help_text=_(
+ "Enable GitHub integration for all users. When enabled, approved_users whitelist is ignored"
+ ),
+ )
+ approved_users = models.ManyToManyField(
+ settings.AUTH_USER_MODEL,
+ blank=True,
+ related_name="github_integration_approved",
+ help_text=_(
+ "Whitelist of users who can access GitHub integration. Superusers/staff are implicitly included"
+ ),
+ )
+
+ def can_use_github_integration(self, user):
+ """
+ Check if a user can use the GitHub integration feature.
+
+ returns True if:
+ - globally enabled (allows anonymous users)
+ - user is superuser or staff
+ - user is in the approved_users
+ """
+ if self.is_enabled_globally:
+ return True
+ if user is None or not user.is_authenticated:
+ return False
+ if user.is_superuser or user.is_staff:
+ return True
+ return self.approved_users.filter(id=user.id).exists()
+
+ @staticmethod
+ def github_app_settings_help_content():
+ """Create formatted content for a configuration help panel on the GitHub Integration settings page"""
+ info_html = format_html(
+ 'See the infra wiki for configuration info.
'
+ )
+ app_name = settings.GITHUB_INTEGRATION_APP_NAME
+ org_name = settings.GITHUB_MODEL_LIBRARY_ORG_NAME
+ warnings = []
+ if not app_name:
+ warnings.append(
+ "GitHub app name is not configured. Set GITHUB_INTEGRATION_APP_NAME in the environment."
+ )
+ if not org_name:
+ warnings.append(
+ "GitHub organization name is not configured. Set GITHUB_MODEL_LIBRARY_ORG_NAME in the environment."
+ )
+ if warnings:
+ help_html = format_html(
+ 'WARNING! {}',
+ " ".join(warnings),
+ )
+ else:
+ url = f"https://github.com/organizations/{org_name}/settings/apps/{slugify(app_name)}"
+ help_html = format_html(
+ '{} settings on GitHub',
+ url,
+ app_name,
+ )
+ return info_html + help_html
+
+ panels = [
+ HelpPanel(content=github_app_settings_help_content()),
+ FieldPanel("is_beta"),
+ FieldPanel("is_enabled_globally"),
+ FieldPanel(
+ "approved_users",
+ widget=FilteredSelectMultiple("approved users", is_stacked=False),
+ ),
+ InlinePanel("faq_entries", label="FAQ Entries"),
+ InlinePanel("existing_repo_info_items", label="Existing Repo Info Items"),
+ InlinePanel(
+ "existing_repo_tutorial_videos", label="Existing Repo Tutorial Videos"
+ ),
+ InlinePanel("new_repo_info_items", label="New Repo Info Items"),
+ InlinePanel("new_repo_tutorial_videos", label="New Repo Tutorial Videos"),
+ ]
diff --git a/django/library/permissions.py b/django/library/permissions.py
index 51d9675a0..4dede205e 100644
--- a/django/library/permissions.py
+++ b/django/library/permissions.py
@@ -1,6 +1,24 @@
-from rest_framework.permissions import DjangoModelPermissions
+from rest_framework.permissions import DjangoModelPermissions, BasePermission
+from rest_framework.exceptions import PermissionDenied as DrfPermissionDenied
+from wagtail.models import Site
from core.permissions import ObjectPermissions
+from library.models import GitHubIntegrationConfiguration
+
+
+class GitHubIntegrationPermission(BasePermission):
+ """Permission class to check if user can use GitHub integration."""
+
+ def has_permission(self, request, view):
+ try:
+ site = Site.objects.get(is_default_site=True)
+ config = GitHubIntegrationConfiguration.for_site(site)
+ except (Site.DoesNotExist, GitHubIntegrationConfiguration.DoesNotExist):
+ raise DrfPermissionDenied("GitHub integration is not available")
+
+ if not config.can_use_github_integration(request.user):
+ raise DrfPermissionDenied("GitHub integration is not available")
+ return True
class CodebaseReleaseUnpublishedFilePermissions(ObjectPermissions):
diff --git a/django/library/serializers.py b/django/library/serializers.py
index 43b493f5e..75efb470a 100644
--- a/django/library/serializers.py
+++ b/django/library/serializers.py
@@ -30,6 +30,9 @@
RelatedUserSerializer,
)
from .models import (
+ CodebaseGitRemote,
+ GitRefSyncState,
+ ImportedReleaseSyncState,
PeerReviewer,
ReleaseContributor,
Codebase,
@@ -466,6 +469,9 @@ class RelatedCodebaseSerializer(serializers.ModelSerializer, FeaturedImageMixin)
Sparse codebase serializer
"""
+ absolute_url = serializers.URLField(source="get_absolute_url", read_only=True)
+ github_config_url = serializers.SerializerMethodField(read_only=True)
+ active_git_remote = serializers.SerializerMethodField(read_only=True)
all_contributors = ContributorSerializer(many=True, read_only=True)
tags = TagSerializer(many=True)
version_number = serializers.ReadOnlyField(source="latest_version.version_number")
@@ -479,6 +485,15 @@ class RelatedCodebaseSerializer(serializers.ModelSerializer, FeaturedImageMixin)
live = serializers.ReadOnlyField()
description = MarkdownField()
+ def get_github_config_url(self, instance):
+ return reverse("library:codebase-git-remotes-list", args=[instance.identifier])
+
+ def get_active_git_remote(self, instance):
+ active_git_remote = instance.active_git_remote
+ if active_git_remote:
+ return CodebaseGitRemoteSerializer(active_git_remote).data
+ return None
+
def create(self, validated_data):
return create(self.Meta.model, validated_data, self.context)
@@ -488,7 +503,10 @@ def update(self, instance, validated_data):
class Meta:
model = Codebase
fields = (
+ "absolute_url",
"all_contributors",
+ "github_config_url",
+ "active_git_remote",
"tags",
"title",
"first_published_at",
@@ -504,6 +522,53 @@ class Meta:
)
+class GitRefSyncStateSerializer(serializers.ModelSerializer):
+ can_push = serializers.SerializerMethodField()
+
+ def get_can_push(self, instance):
+ return instance.can_push()
+
+ class Meta:
+ model = GitRefSyncState
+ fields = "__all__"
+
+
+class ImportedReleaseSyncStateSerializer(serializers.ModelSerializer):
+ can_reimport = serializers.SerializerMethodField()
+
+ def get_can_reimport(self, instance):
+ return instance.can_reimport()
+
+ class Meta:
+ model = ImportedReleaseSyncState
+ fields = "__all__"
+
+
+class CodebaseGitRemoteSerializer(serializers.ModelSerializer):
+ is_active = serializers.ReadOnlyField()
+
+ class Meta:
+ model = CodebaseGitRemote
+ fields = (
+ "id",
+ "owner",
+ "repo_name",
+ "url",
+ "is_user_repo",
+ "is_preexisting",
+ "is_active",
+ )
+ read_only_fields = (
+ "id",
+ "owner",
+ "repo_name",
+ "url",
+ "is_user_repo",
+ "is_preexisting",
+ "is_active",
+ )
+
+
class CodebaseImageSerializer(serializers.ModelSerializer):
identifier = serializers.IntegerField(source="id")
name = serializers.CharField(source="title")
@@ -596,6 +661,7 @@ class CodebaseReleaseSerializer(serializers.ModelSerializer):
release_notes = MarkdownField(max_length=2048)
urls = serializers.SerializerMethodField()
review_status = serializers.SerializerMethodField()
+ imported_release_sync_state = ImportedReleaseSyncStateSerializer(read_only=True)
def get_urls(self, instance):
request_peer_review_url = instance.get_request_peer_review_url()
@@ -617,6 +683,7 @@ class Meta:
model = CodebaseRelease
fields = (
"absolute_url",
+ "status",
"can_edit_originals",
"citation_text",
"release_contributors",
@@ -645,11 +712,21 @@ class Meta:
"output_data_url",
"version_number",
"id",
+ "imported_release_sync_state",
"share_url",
"urls",
)
+class CodebaseReleaseWithGitRefSyncStateSerializer(CodebaseReleaseSerializer):
+ """Extends base release serializer to include git ref sync state for local mirror context"""
+
+ git_ref_sync_state = GitRefSyncStateSerializer(read_only=True)
+
+ class Meta(CodebaseReleaseSerializer.Meta):
+ fields = CodebaseReleaseSerializer.Meta.fields + ("git_ref_sync_state",)
+
+
class CodebaseReleaseEditSerializer(CodebaseReleaseSerializer):
possible_licenses = serializers.SerializerMethodField()
diff --git a/django/library/tasks.py b/django/library/tasks.py
index 0cbc93804..911bead15 100644
--- a/django/library/tasks.py
+++ b/django/library/tasks.py
@@ -1,4 +1,15 @@
from huey.contrib.djhuey import db_task, on_commit_task
+from django.conf import settings
+
+from .models import (
+ Codebase,
+ CodebaseGitRemote,
+ CodebaseRelease,
+ GitRefSyncState,
+ ImportedReleaseSyncState,
+)
+from .github_integration import GitHubApi
+from .fs import CodebaseGitRepositoryApi
import logging
@@ -6,12 +17,141 @@
@db_task(retries=1, retry_delay=30)
-def update_fs_release_metadata(release_id: int):
- from .models import CodebaseRelease
+def build_local_git_repo(codebase_id: int):
+ """Build or update the local git mirror repository for a codebase without pushing.
+
+ Safe to call multiple times: will append new releases if mirror exists and has prior build state,
+ otherwise rebuilds from scratch.
+ """
+ codebase = Codebase.objects.get(id=codebase_id)
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ # update_or_build handles both first-time build and incremental append
+ git_fs_api.update_or_build()
+
+
+@db_task(retries=1, retry_delay=30)
+def append_releases_to_local_git_repo(codebase_id: int):
+ """Appends local releases to git mirror."""
+ codebase = Codebase.objects.get(id=codebase_id)
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ git_fs_api.append_releases()
+
+
+@db_task(retries=1, retry_delay=30)
+def import_github_release_task(
+ codebase_id: int,
+ remote_id: int,
+ github_release_id: str,
+ custom_version: str | None = None,
+):
+ """Asynchronously import a GitHub release for a given remote using pre-fetched metadata stored on the sync state.
+
+ The sync state should have been created and populated in the API layer.
+ On failure, marks the state as ERROR.
+ """
+ from .github_integration import GitHubReleaseImporter
+
+ remote = CodebaseGitRemote.objects.get(id=remote_id, codebase__id=codebase_id)
+
+ try:
+ # sync state must exist already
+ sync_state = ImportedReleaseSyncState.objects.filter(
+ remote=remote, github_release_id=str(github_release_id)
+ ).first()
+ if not sync_state:
+ raise ValueError("Missing sync state for GitHub release import")
+ importer = GitHubReleaseImporter(
+ remote=remote, github_release_id=str(github_release_id)
+ )
+ importer.import_or_reimport(custom_version=custom_version)
+ except Exception as e:
+ try:
+ sync_state = ImportedReleaseSyncState.objects.filter(
+ remote=remote, github_release_id=str(github_release_id)
+ ).first()
+ if sync_state:
+ sync_state.log_failure(str(e))
+ except Exception:
+ logger.exception("Failed to update sync state after import error")
+ logger.exception("Failed to import GitHub release asynchronously: %s", e)
+
+
+@db_task(retries=1, retry_delay=30)
+def update_local_repo_release_branch(release_id: int):
+ """asynchronous task that updates a SINGLE RELEASE BRANCH with any metadata changes
+ that may have occurred.
+
+ This should be called when release metadata has been changed
+ """
+ release = CodebaseRelease.objects.get(id=release_id)
+ codebase = release.codebase
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ local_repo = git_fs_api.update_release_branch(release)
+
+
+@db_task(retries=1, retry_delay=30)
+def push_all_releases_to_github(codebase_id: int, remote_id: int):
+ """Manually push all releases to GitHub for a remote, one-by-one."""
+ codebase = Codebase.objects.get(id=codebase_id)
+ remote = CodebaseGitRemote.objects.get(id=remote_id, codebase__id=codebase_id)
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ # ensure local repo is up-to-date before pushing
+ local_repo = git_fs_api.update_or_build()
+ gh_api = GitHubApi(codebase=codebase, local_repo=local_repo, remote=remote)
+ # ensure remote exists and persist url
+ repo = gh_api.get_or_create_repo()
+ if remote.url != repo.html_url:
+ remote.url = repo.html_url
+ remote.save(update_fields=["url", "last_modified"])
+
+ # push main first if job started
+ main_state = GitRefSyncState.for_codebase(codebase)
+ if main_state.status == GitRefSyncState.Status.RUNNING:
+ try:
+ commit_sha, summary = gh_api.push_main(local_repo)
+ main_state.record_push_success(commit_sha, summary)
+ except Exception as e:
+ main_state.log_failure(str(e))
+ logger.exception("Failed to push main: %s", e)
+
+ # iterate releases in ascending semver order
+ releases = codebase.ordered_releases_list(internal_only=True, asc=True)
+ for release in releases:
+ state = release.get_or_create_git_ref_sync_state()
+ # skip if job not started
+ if state.status != GitRefSyncState.Status.RUNNING:
+ continue
+ try:
+ commit_sha, summary = gh_api.push_release(
+ local_repo,
+ release,
+ branch_name=state.branch_name,
+ tag_name=state.tag_name,
+ )
+ # create a release for this tag on GitHub if needed
+ gh_api.create_release_for_tag(local_repo, release.version_number)
+ state.record_push_success(commit_sha, summary)
+ except Exception as e:
+ state.log_failure(str(e))
+ # continue with remaining releases, keeping per-release isolation
+ logger.exception("Failed to push release %s: %s", release.version_number, e)
+ continue
+
+
+@db_task(retries=1, retry_delay=30)
+def update_fs_release_metadata(release_id: int):
release = CodebaseRelease.objects.get(id=release_id)
+ codebase = release.codebase
fs_api = release.get_fs_api()
- fs_api.rebuild(metadata_only=True)
+ fs_api.rebuild_metadata()
+ # if the release is published and there is a git ref sync state for this release, update it with
+ # a new commit
+ if (
+ release.is_published
+ and getattr(release, "git_ref_sync_state", None)
+ ):
+ update_local_repo_release_branch(release_id)
@on_commit_task()
diff --git a/django/library/tests/base.py b/django/library/tests/base.py
index 001478b69..7ea8bb20a 100644
--- a/django/library/tests/base.py
+++ b/django/library/tests/base.py
@@ -1,10 +1,11 @@
import io
import logging
+from pathlib import Path
import random
from uuid import UUID
from core.tests.base import ContentModelFactory, UserFactory
-from library.fs import FileCategoryDirectories
+from library.fs import FileCategories
from library.models import (
Codebase,
CodebaseRelease,
@@ -19,6 +20,9 @@
)
from library.serializers import CodebaseSerializer
+TEST_SAMPLES_DIR = Path("library/tests/samples")
+
+
logger = logging.getLogger(__name__)
@@ -179,7 +183,7 @@ class ReleaseSetup:
PROGRAMMING_LANGUAGES = ["Python", "TypeScript"]
@classmethod
- def setUpPublishableDraftRelease(cls, codebase):
+ def setUpPublishableDraftRelease(cls, codebase, with_files=True):
draft_release = codebase.create_release(
status=CodebaseRelease.Status.DRAFT,
initialize=True,
@@ -196,15 +200,14 @@ def setUpPublishableDraftRelease(cls, codebase):
release_contributor_factory = ReleaseContributorFactory(draft_release)
contributor = contributor_factory.create()
release_contributor_factory.create(contributor)
-
- code_file = io.BytesIO(b"print('hello world')")
- code_file.name = "some_code_file.py"
- docs_file = io.BytesIO(b"# Documentation")
- docs_file.name = "some_doc_file.md"
- fs_api = draft_release.get_fs_api()
- fs_api.add(content=code_file, category=FileCategoryDirectories.code)
- fs_api.add(content=docs_file, category=FileCategoryDirectories.docs)
+ if with_files:
+ code_file = io.BytesIO(b"print('hello world')")
+ code_file.name = "some_code_file.py"
+ docs_file = io.BytesIO(b"# Documentation")
+ docs_file.name = "some_doc_file.md"
+ fs_api = draft_release.get_fs_api()
+ fs_api.add(content=code_file, category=FileCategories.code)
+ fs_api.add(content=docs_file, category=FileCategories.docs)
draft_release.save()
-
return draft_release
diff --git a/django/library/tests/archives/.gitignore b/django/library/tests/samples/archives/.gitignore
similarity index 100%
rename from django/library/tests/archives/.gitignore
rename to django/library/tests/samples/archives/.gitignore
diff --git a/django/library/tests/archives/invalid.zip b/django/library/tests/samples/archives/invalid.zip
similarity index 100%
rename from django/library/tests/archives/invalid.zip
rename to django/library/tests/samples/archives/invalid.zip
diff --git a/django/library/tests/archives/nestedcode/.DS_Store b/django/library/tests/samples/archives/nestedcode/.DS_store
similarity index 100%
rename from django/library/tests/archives/nestedcode/.DS_Store
rename to django/library/tests/samples/archives/nestedcode/.DS_store
diff --git a/django/library/tests/archives/nestedcode/.svn/svn_files_here b/django/library/tests/samples/archives/nestedcode/.svn/svn_files_here
similarity index 100%
rename from django/library/tests/archives/nestedcode/.svn/svn_files_here
rename to django/library/tests/samples/archives/nestedcode/.svn/svn_files_here
diff --git a/django/library/tests/archives/nestedcode/README.md b/django/library/tests/samples/archives/nestedcode/README.md
similarity index 100%
rename from django/library/tests/archives/nestedcode/README.md
rename to django/library/tests/samples/archives/nestedcode/README.md
diff --git a/django/library/tests/archives/nestedcode/src/ex.py b/django/library/tests/samples/archives/nestedcode/src/ex.py
similarity index 100%
rename from django/library/tests/archives/nestedcode/src/ex.py
rename to django/library/tests/samples/archives/nestedcode/src/ex.py
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py b/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py
new file mode 100644
index 000000000..8cde7829c
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py
@@ -0,0 +1 @@
+print("hello world")
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv b/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv
new file mode 100644
index 000000000..29aa17e68
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv
@@ -0,0 +1,2 @@
+horses,sheep
+10,15
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt b/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt
new file mode 100644
index 000000000..c9be3ff53
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt
@@ -0,0 +1,3 @@
+instructions:
+
+python model.py
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt b/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt
new file mode 100644
index 000000000..29fee6381
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt
@@ -0,0 +1 @@
+result tells us there are more sheep
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py
new file mode 100644
index 000000000..f4963717d
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py
@@ -0,0 +1,3 @@
+class Cow:
+ def __init__(self, name):
+ self.name = name
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py
new file mode 100644
index 000000000..84790eaf9
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py
@@ -0,0 +1,3 @@
+class Horse:
+ def __init__(self, name):
+ self.name = name
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py
new file mode 100644
index 000000000..3c575356d
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py
@@ -0,0 +1,3 @@
+class Sheep:
+ def __init__(self, name):
+ self.name = name
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py
new file mode 100644
index 000000000..8cde7829c
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py
@@ -0,0 +1 @@
+print("hello world")
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv b/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv
new file mode 100644
index 000000000..603cf0ed3
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv
@@ -0,0 +1,2 @@
+horses,sheep,cows
+10,15,5
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md b/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md
new file mode 100644
index 000000000..64258c3b0
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md
@@ -0,0 +1,5 @@
+# instructions:
+
+```
+python model.py
+```
\ No newline at end of file
diff --git a/django/library/tests/test_fs.py b/django/library/tests/test_fs.py
index 5bcd21f9a..b92863816 100644
--- a/django/library/tests/test_fs.py
+++ b/django/library/tests/test_fs.py
@@ -1,32 +1,38 @@
from pathlib import Path
-
+from git import Repo
from django.test import TestCase
+from django.conf import settings
from core.tests.base import (
UserFactory,
destroy_test_shared_folders,
initialize_test_shared_folders,
+ clear_test_shared_folder,
)
from library.fs import (
- FileCategoryDirectories,
+ FileCategories,
StagingDirectories,
MessageLevels,
import_archive,
+ CodebaseGitRepositoryApi,
)
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
+from library.models import License, GitRefSyncState, ProgrammingLanguage
import logging
logger = logging.getLogger(__name__)
+CODEBASE_CONTENT_CATEGORIES = ["code", "data", "docs"]
+
def setUpModule():
initialize_test_shared_folders()
class ArchiveExtractorTestCase(TestCase):
- nested_code_folder = Path("library/tests/archives/nestedcode")
+ nested_code_folder = TEST_SAMPLES_DIR / "archives" / "nestedcode"
def setUp(self):
self.user_factory = UserFactory()
@@ -47,44 +53,200 @@ def test_zipfile_saving(self):
self.assertEqual(len(logs), 2)
self.assertEqual(
set(
- fs_api.list(StagingDirectories.originals, FileCategoryDirectories.code)
+ fs_api.list(StagingDirectories.originals, FileCategories.code)
),
{"nestedcode.zip"},
)
# Notice that .DS_Store and .svn folder file are eliminated
self.assertEqual(
- set(fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)),
+ set(fs_api.list(StagingDirectories.sip, FileCategories.code)),
{"src/ex.py", "README.md"},
)
fs_api.get_or_create_sip_bag(self.codebase_release.bagit_info)
- fs_api.clear_category(FileCategoryDirectories.code)
+ fs_api.clear_category(FileCategories.code)
self.assertEqual(
set(
- fs_api.list(StagingDirectories.originals, FileCategoryDirectories.code)
+ fs_api.list(StagingDirectories.originals, FileCategories.code)
),
set(),
)
self.assertEqual(
- set(fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)),
+ set(fs_api.list(StagingDirectories.sip, FileCategories.code)),
set(),
)
def test_invalid_zipfile_saving(self):
- archive_name = "library/tests/archives/invalid.zip"
+ archive_name = str(TEST_SAMPLES_DIR / "archives" / "invalid.zip")
fs_api = self.codebase_release.get_fs_api()
with open(archive_name, "rb") as f:
msgs = fs_api.add(
- FileCategoryDirectories.code, content=f, name="invalid.zip"
+ FileCategories.code, content=f, name="invalid.zip"
)
logs, level = msgs.serialize()
self.assertEqual(level, MessageLevels.error)
self.assertEqual(len(logs), 1)
+ def tearDown(self):
+ clear_test_shared_folder(settings.REPOSITORY_ROOT)
+
@classmethod
def tearDownClass(cls):
super().tearDownClass()
cls.nested_code_folder.with_suffix(".zip").unlink(missing_ok=True)
+class GitRepoApiTestCase(TestCase):
+ model_dir = TEST_SAMPLES_DIR / "releases" / "animals-model"
+ release_1_dir = model_dir / "1.0.0"
+ release_2_dir = model_dir / "2.0.0"
+
+ def setUp(self):
+ self.user_factory = UserFactory()
+ self.submitter = self.user_factory.create()
+ self.codebase_factory = CodebaseFactory(submitter=self.submitter)
+ self.codebase = self.codebase_factory.create()
+ self.release_1 = self.codebase.create_release()
+
+ def tearDown(self):
+ clear_test_shared_folder(settings.REPOSITORY_ROOT)
+
+ def _verify_git_repo_status(self, api, release_count):
+ """Verify git repo structure and contents are correct."""
+ repo = Repo(api.repo_dir)
+ self.assertFalse(repo.is_dirty())
+ self.assertEqual(sum(1 for _ in repo.iter_commits()), release_count)
+ self.assertEqual(len(repo.tags), release_count)
+ # check contents
+ self.assertTrue((api.repo_dir / "codemeta.json").exists())
+ self.assertTrue((api.repo_dir / "CITATION.cff").exists())
+ self.assertTrue((api.repo_dir / "LICENSE").exists())
+
+ def test_repo_build(self):
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_1.publish()
+ public_release_count = self.codebase.releases.public().count()
+ self.assertEqual(public_release_count, 1)
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ # check that the git ref sync states are created and the repo is built
+ built_releases_count = GitRefSyncState.objects.filter(
+ release__codebase=self.codebase, built_commit_sha__isnull=False
+ ).count()
+ self.assertEqual(built_releases_count, 1)
+ self.assertTrue(api.repo_dir.exists())
+ self._verify_git_repo_status(api, public_release_count)
+ fs_api = self.release_1.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ self.assertTrue(
+ api.dirs_equal(
+ fs_api.sip_contents_dir / category,
+ api.repo_dir / category,
+ )
+ )
+
+ def test_repo_append_releases(self):
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_1.publish()
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ self.release_2 = self.codebase.create_release()
+ update_release_from_sample(
+ self.release_2, self.release_2_dir, version_number="2.0.0"
+ )
+ self.release_2.publish()
+ api.append_releases()
+
+ built_releases_count = GitRefSyncState.objects.filter(
+ release__codebase=self.codebase, built_commit_sha__isnull=False
+ ).count()
+ self.assertEqual(built_releases_count, 2)
+ public_release_count = self.codebase.releases.public().count()
+ self._verify_git_repo_status(api, public_release_count)
+ fs_api = self.release_2.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ self.assertTrue(
+ api.dirs_equal(
+ fs_api.sip_contents_dir / category,
+ api.repo_dir / category,
+ )
+ )
+
+ def test_will_not_append_lower_version(self):
+ # publish 2.0.0 first and build
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_2 = self.codebase.create_release()
+ update_release_from_sample(
+ self.release_2, self.release_2_dir, version_number="2.0.0"
+ )
+ self.release_2.publish()
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ # now publish release 1.0.0
+ self.release_1.publish()
+ self.assertRaises(ValueError, api.append_releases)
+
+ def test_repo_rebuild(self):
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_1.publish()
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ self.release_2 = self.codebase.create_release()
+ update_release_from_sample(
+ self.release_2, self.release_2_dir, version_number="2.0.0"
+ )
+ self.release_2.publish()
+ api.build()
+
+ built_releases_count = GitRefSyncState.objects.filter(
+ release__codebase=self.codebase, built_commit_sha__isnull=False
+ ).count()
+ self.assertEqual(built_releases_count, 2)
+ public_release_count = self.codebase.releases.public().count()
+ self._verify_git_repo_status(api, public_release_count)
+ fs_api = self.release_2.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ self.assertTrue(
+ api.dirs_equal(
+ fs_api.sip_contents_dir / category,
+ api.repo_dir / category,
+ )
+ )
+
+
def tearDownModule():
destroy_test_shared_folders()
+
+
+# helpers ================================================
+
+
+def upload_category(fs_api, release_dir: Path, category: str):
+ category_path = release_dir / category
+ for filepath in category_path.rglob("*"):
+ if filepath.is_file():
+ with filepath.open("rb") as f:
+ relpath = filepath.relative_to(category_path)
+ file_name = str(relpath)
+ fs_api.add(FileCategories[category], content=f, name=file_name)
+
+
+def update_release_from_sample(release, sample_dir, version_number):
+ release.os = "Linux"
+ python, _created = ProgrammingLanguage.objects.get_or_create_by_name("Python")
+ release.programming_languages.add(python)
+ release.license = License.objects.create(name="MIT")
+ release.release_notes = "Initial release"
+ release.version_number = version_number
+ release.save()
+ fs_api = release.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ upload_category(fs_api, sample_dir, category)
+ return release
diff --git a/django/library/tests/test_github_integration.py b/django/library/tests/test_github_integration.py
new file mode 100644
index 000000000..1092f4cad
--- /dev/null
+++ b/django/library/tests/test_github_integration.py
@@ -0,0 +1,377 @@
+from unittest.mock import patch, MagicMock
+from django.test import TestCase
+from github.GithubException import UnknownObjectException
+
+from core.tests.base import UserFactory
+from library.github_integration import (
+ GitHubApi,
+ GitHubRepoValidator,
+ GitHubReleaseImporter,
+ _serialize_github_release_listing_item,
+ extract_semver,
+)
+from library.models import (
+ CodebaseGitRemote,
+ GithubIntegrationAppInstallation,
+ CodebaseRelease,
+ ImportedReleaseSyncState,
+)
+from library.tests.base import CodebaseFactory
+
+SAMPLE_PAYLOAD = {
+ "action": "released",
+ "release": {
+ "id": 12345,
+ "tag_name": "v1.0.0",
+ "name": "Version 1.0.0",
+ "body": "Initial release",
+ "draft": False,
+ "prerelease": False,
+ "zipball_url": "https://api.github.com/repos/testuser/test-repo/zipball/v1.0.0",
+ "html_url": "https://github.com/testuser/test-repo/releases/tag/v1.0.0",
+ },
+ "repository": {
+ "name": "test-repo",
+ "owner": {"login": "testuser"},
+ "private": False,
+ },
+ "installation": {"id": 54321},
+}
+
+
+class GitHubRepoValidatorTests(TestCase):
+ def setUp(self):
+ # set up a user and installation for testing validator methods
+ self.user = UserFactory().create()
+ self.installation = GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ github_login="testuser",
+ installation_id=1,
+ github_user_id=123,
+ )
+
+ def test_validate_format(self):
+ # test valid repo names
+ for name in ["repo", "repo-1", "repo.1", "my_repo"]:
+ with self.subTest(name=name):
+ validator = GitHubRepoValidator(name)
+ self.assertIsNone(validator.validate_format())
+
+ # test invalid repo names
+ for name in [
+ "repo with space",
+ "repo!",
+ "a" * 101,
+ "repo.git",
+ "contains-github",
+ ]:
+ with self.subTest(name=name):
+ validator = GitHubRepoValidator(name)
+ with self.assertRaises(ValueError):
+ validator.validate_format()
+
+
+class GitHubReleaseImporterTests(TestCase):
+ def setUp(self):
+ # set up a user, codebase, and remote for importing releases
+ self.user = UserFactory().create()
+ self.installation = GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ github_login="testuser",
+ installation_id=54321,
+ github_user_id=123,
+ )
+ self.codebase = CodebaseFactory(submitter=self.user).create()
+ self.remote = CodebaseGitRemote.objects.create(
+ codebase=self.codebase,
+ owner="testuser",
+ repo_name="test-repo",
+ is_user_repo=True,
+ is_active=True,
+ )
+ self.payload = SAMPLE_PAYLOAD.copy()
+ # create sync state from payload
+ self.sync_state = ImportedReleaseSyncState.for_github_release(
+ self.remote, self.payload["release"]
+ )
+
+ def create_importer(self, github_release_id="12345"):
+ return GitHubReleaseImporter(self.remote, github_release_id)
+
+ def test_init_success(self):
+ # should initialize successfully with a valid remote and release id
+ importer = self.create_importer("12345")
+ self.assertEqual(importer.github_release_id, "12345")
+ self.assertEqual(importer.codebase, self.codebase)
+
+ def test_init_failures(self):
+ # test various invalid scenarios that should raise ValueError
+ # missing sync state
+ with self.assertRaises(ValueError):
+ self.create_importer("99999")
+
+ # sync state without download_url
+ ImportedReleaseSyncState.objects.create(
+ remote=self.remote,
+ github_release_id="88888",
+ download_url="", # empty download url
+ )
+ with self.assertRaises(ValueError):
+ self.create_importer("88888")
+
+ def test_extract_semver(self):
+ # delegate test, see ExtractSemverTests for exhaustive coverage
+ importer = self.create_importer("12345")
+ self.assertEqual(importer.extract_semver("v1.2.3"), "1.2.3")
+
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_import_new_release(self, mock_get_token, mock_get_fs_api, mock_get_repo):
+ # mock token and fs_api calls
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {"name": "test-repo", "full_name": "testuser/test-repo"}
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {}) # codemeta, cff
+ mock_get_fs_api.return_value = mock_fs_api
+
+ # import a new release
+ importer = self.create_importer("12345")
+ success = importer.import_new_release()
+
+ # check that it was successful and objects were created
+ self.assertTrue(success)
+ self.assertTrue(
+ CodebaseRelease.objects.filter(
+ codebase=self.codebase, version_number="1.0.0"
+ ).exists()
+ )
+ release = CodebaseRelease.objects.get(version_number="1.0.0")
+ self.assertEqual(release.imported_release_sync_state.github_release_id, "12345")
+ self.assertEqual(release.submitter, self.codebase.submitter)
+ mock_fs_api.import_release_package.assert_called_once()
+
+ @patch("library.github_integration.GitHubApi.get_release_raw_for_remote")
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_reimport_release(self, mock_get_token, mock_get_fs_api, mock_get_repo, mock_get_release):
+ # mock token and fs_api calls
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {"name": "test-repo", "full_name": "testuser/test-repo"}
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {}) # codemeta, cff
+ mock_get_fs_api.return_value = mock_fs_api
+
+ # first, import a new release
+ importer = self.create_importer("12345")
+ importer.import_or_reimport()
+
+ self.assertEqual(CodebaseRelease.objects.count(), 1)
+ release = CodebaseRelease.objects.first()
+ self.assertEqual(
+ release.imported_release_sync_state.download_url,
+ "https://api.github.com/repos/testuser/test-repo/zipball/v1.0.0",
+ )
+
+ # now, create a new importer with an updated payload for an "edited" event
+ reimport_payload = self.payload.copy()
+ reimport_payload["action"] = "edited"
+ reimport_payload["release"] = reimport_payload["release"].copy()
+ new_url = (
+ "https://api.github.com/repos/testuser/test-repo/zipball/v1.0.0-updated"
+ )
+ reimport_payload["release"]["zipball_url"] = new_url
+ # mock get_release_raw_for_remote to return the updated payload
+ mock_get_release.return_value = reimport_payload["release"]
+
+ importer2 = self.create_importer("12345")
+ success = importer2.import_or_reimport()
+
+ # assert that the re-import was successful and the release was updated
+ self.assertTrue(success)
+ self.assertEqual(mock_fs_api.import_release_package.call_count, 2)
+
+ release.refresh_from_db()
+ self.assertEqual(release.imported_release_sync_state.download_url, new_url)
+ # release version number should NOT have changed
+ self.assertEqual(release.version_number, "1.0.0")
+
+ @patch("library.github_integration.GitHubApi.get_release_raw_for_remote")
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_reimport_skips_when_download_url_unchanged(
+ self, mock_get_token, mock_get_fs_api, mock_get_repo, mock_get_release
+ ):
+ # when the zipball URL has not changed, reimport_release should detect no
+ # changes and return without calling import_release_package
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {"name": "test-repo", "full_name": "testuser/test-repo"}
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {})
+ mock_get_fs_api.return_value = mock_fs_api
+
+ # perform an initial import so a linked release exists
+ importer = self.create_importer("12345")
+ first_success = importer.import_or_reimport()
+ self.assertTrue(first_success)
+ self.assertEqual(mock_fs_api.import_release_package.call_count, 1)
+
+ # mock get_release to return the same zipball_url as the cached sync state
+ original_url = self.sync_state.download_url
+ mock_get_release.return_value = {
+ "id": 12345,
+ "tag_name": "v1.0.0",
+ "name": "Version 1.0.0",
+ "zipball_url": original_url,
+ "html_url": "https://github.com/testuser/test-repo/releases/tag/v1.0.0",
+ }
+
+ importer2 = self.create_importer("12345")
+ success = importer2.import_or_reimport()
+
+ # import_release_package must NOT be called a second time
+ self.assertTrue(success)
+ self.assertEqual(mock_fs_api.import_release_package.call_count, 1)
+
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_import_new_release_version_collision(
+ self, mock_get_token, mock_get_fs_api, mock_get_repo
+ ):
+ # even with valid metadata/api mocks, import should fail when version already exists
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {
+ "name": "test-repo",
+ "full_name": "testuser/test-repo",
+ }
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {})
+ mock_get_fs_api.return_value = mock_fs_api
+
+ existing_release = self.codebase.create_release()
+ existing_release.version_number = "1.0.0"
+ existing_release.save()
+
+ importer = self.create_importer("12345")
+ success = importer.import_new_release()
+
+ self.assertFalse(success)
+ self.assertEqual(
+ CodebaseRelease.objects.filter(codebase=self.codebase, version_number="1.0.0").count(),
+ 1,
+ )
+ mock_fs_api.import_release_package.assert_not_called()
+
+
+class ExtractSemverTests(TestCase):
+ def test_standard_version(self):
+ self.assertEqual(extract_semver("1.2.3"), "1.2.3")
+
+ def test_leading_v(self):
+ self.assertEqual(extract_semver("v1.2.3"), "1.2.3")
+
+ def test_embedded_in_string(self):
+ self.assertEqual(extract_semver("version 1.2.3-beta"), "1.2.3")
+
+ def test_two_part_version_returns_none(self):
+ self.assertIsNone(extract_semver("1.2"))
+
+ def test_non_version_string_returns_none(self):
+ self.assertIsNone(extract_semver("invalid-version"))
+
+ def test_empty_string_returns_none(self):
+ self.assertIsNone(extract_semver(""))
+
+ def test_non_string_returns_none(self):
+ self.assertIsNone(extract_semver(None))
+ self.assertIsNone(extract_semver(123))
+
+ def test_oversized_input_returns_none(self):
+ self.assertIsNone(extract_semver("1.2.3" * 300))
+
+
+class GitHubIntegrationHelpersTests(TestCase):
+ def test_serialize_github_release_listing_item_prefers_raw_data(self):
+ release = MagicMock()
+ release.id = 999
+ release.raw_data = {
+ "id": 12345,
+ "tag_name": "v1.2.3",
+ "name": "Release 1.2.3",
+ "html_url": "https://github.com/acme/repo/releases/tag/v1.2.3",
+ "zipball_url": "https://api.github.com/repos/acme/repo/zipball/v1.2.3",
+ "draft": False,
+ "prerelease": False,
+ "created_at": "2025-01-01T00:00:00Z",
+ "published_at": "2025-01-02T00:00:00Z",
+ }
+
+ data, raw = _serialize_github_release_listing_item(release)
+
+ self.assertEqual(raw["id"], 12345)
+ self.assertEqual(data["id"], "12345")
+ self.assertEqual(data["tag_name"], "v1.2.3")
+ self.assertEqual(data["version"], "1.2.3")
+ self.assertTrue(data["has_semantic_versioning"])
+
+ def test_serialize_github_release_listing_item_falls_back_to_attributes(self):
+ release = MagicMock()
+ release.raw_data = {}
+ release.id = 555
+ release.tag_name = "2.0.1"
+ release.title = None
+ release.name = "Project Release 2.0.1"
+ release.html_url = "https://github.com/acme/repo/releases/tag/2.0.1"
+ release.zipball_url = "https://api.github.com/repos/acme/repo/zipball/2.0.1"
+ release.draft = True
+ release.prerelease = False
+ release.created_at = "2025-03-01T00:00:00Z"
+ release.published_at = "2025-03-02T00:00:00Z"
+
+ data, _ = _serialize_github_release_listing_item(release)
+
+ self.assertEqual(data["id"], "555")
+ self.assertEqual(data["name"], "Project Release 2.0.1")
+ self.assertEqual(data["version"], "2.0.1")
+ self.assertTrue(data["has_semantic_versioning"])
+ self.assertTrue(data["draft"])
+
+ def test_create_release_for_tag_skips_when_release_exists(self):
+ api = GitHubApi.__new__(GitHubApi)
+ api._github_repo = MagicMock()
+ local_repo = MagicMock()
+
+ api._github_repo.get_release.return_value = MagicMock()
+
+ api.create_release_for_tag(local_repo, "v1.0.0")
+
+ api._github_repo.get_release.assert_called_once_with("v1.0.0")
+ api._github_repo.create_git_release.assert_not_called()
+
+ def test_create_release_for_tag_creates_with_tag_message(self):
+ api = GitHubApi.__new__(GitHubApi)
+ api._github_repo = MagicMock()
+ local_repo = MagicMock()
+
+ api._github_repo.get_release.side_effect = UnknownObjectException(
+ 404, {"message": "Not Found"}, None
+ )
+ tag = MagicMock()
+ tag.name = "v1.0.0"
+ tag.commit.message = "Release notes"
+ local_repo.tags = [tag]
+
+ api.create_release_for_tag(local_repo, "v1.0.0")
+
+ api._github_repo.create_git_release.assert_called_once_with(
+ "v1.0.0",
+ name="v1.0.0",
+ message="Release notes",
+ draft=False,
+ prerelease=False,
+ )
+
+
diff --git a/django/library/tests/test_metadata.py b/django/library/tests/test_metadata.py
index dadf08e05..d94155804 100644
--- a/django/library/tests/test_metadata.py
+++ b/django/library/tests/test_metadata.py
@@ -1,11 +1,13 @@
import logging
+from codemeticulous.cff.models import LicenseEnum
+from django.test import TestCase
from core.tests.base import BaseModelTestCase, UserFactory
from .base import (
CodebaseFactory,
ReleaseSetup,
)
-from library.metadata import CodeMeta
+from library.metadata import CodeMeta, ReleaseMetadataConverter
logger = logging.getLogger(__name__)
@@ -90,4 +92,199 @@ def recurse_contents(contents):
recurse_contents(item["contents"])
recurse_contents(sip_contents["contents"])
- self.assertEqual(len(files_found), 3)
+ self.assertEqual(
+ set(label.lower() for label in files_found),
+ {"license", "citation.cff", "codemeta.json"},
+ )
+
+
+class ReleaseMetadataConverterTestCase(TestCase):
+
+ def setUp(self):
+ self.minimal_codemeta = {"name": "Test Code"}
+ self.minimal_cff = {
+ "cff-version": "1.2.0",
+ "title": "Test Title",
+ "message": "Test Message",
+ "authors": [{"name": "Test Author"}],
+ }
+
+ def test_extract_license_from_github(self):
+ converter = ReleaseMetadataConverter(
+ github_repository={"license": {"spdx_id": "MIT"}}
+ )
+ self.assertEqual(converter._extract_license_from_github(), "MIT")
+
+ def test_extract_license_from_cff(self):
+ cff = {**self.minimal_cff, "license": LicenseEnum.GPL_3_0_only}
+ converter = ReleaseMetadataConverter(cff=cff)
+ self.assertEqual(converter._extract_license_from_cff(), "GPL-3.0-only")
+
+ def test_extract_release_notes_from_codemeta(self):
+ codemeta = {**self.minimal_codemeta, "releaseNotes": "From CodeMeta"}
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(
+ converter._extract_release_notes_from_codemeta(), "From CodeMeta"
+ )
+
+ def test_extract_release_notes_from_github(self):
+ converter = ReleaseMetadataConverter(github_release={"body": "From GitHub"})
+ self.assertEqual(converter._extract_release_notes_from_github(), "From GitHub")
+
+ def test_extract_os_from_codemeta(self):
+ codemeta = {**self.minimal_codemeta, "operatingSystem": "Linux"}
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(converter._extract_os_from_codemeta(), "linux")
+
+ def test_extract_os_from_codemeta_variants(self):
+ cases = [
+ ("macOS Sonoma", "macos"),
+ ("Windows 11", "windows"),
+ ("platform independent", "platform_independent"),
+ ("Plan9", "other"),
+ ]
+ for os_value, expected in cases:
+ with self.subTest(os_value=os_value):
+ converter = ReleaseMetadataConverter(
+ codemeta={**self.minimal_codemeta, "operatingSystem": os_value}
+ )
+ self.assertEqual(converter._extract_os_from_codemeta(), expected)
+
+ def test_extract_os_from_codemeta_structured_object(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "operatingSystem": {"name": "Any"},
+ "releaseNotes": "From CodeMeta",
+ }
+ converter = ReleaseMetadataConverter(
+ codemeta=codemeta,
+ github_repository={"license": {"spdx_id": "MIT"}, "language": "Python"},
+ github_release={"body": "From GitHub"},
+ )
+ # Unsupported structured OS input should be ignored without breaking conversion.
+ self.assertEqual(converter._extract_os_from_codemeta(), "")
+ result = converter.convert()
+ self.assertEqual(result["os"], "")
+ self.assertEqual(result["release_notes"], "From GitHub")
+ self.assertEqual(result["license_spdx_id"], "MIT")
+ self.assertEqual(result["programming_languages"], ["Python"])
+
+ def test_extract_programming_languages_from_codemeta(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "programmingLanguage": ["Python", "R"],
+ }
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(
+ converter._extract_programming_languages_from_codemeta(), ["Python", "R"]
+ )
+
+ def test_extract_programming_languages_from_codemeta_edge_cases(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "programmingLanguage": [{"name": "Julia"}, {"foo": "bar"}, 123],
+ }
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ languages = converter._extract_programming_languages_from_codemeta()
+ self.assertEqual(languages, ["Julia", "123"])
+ checked_languages = languages or []
+ self.assertNotIn("bar", checked_languages)
+
+ def test_extract_programming_languages_from_github(self):
+ converter = ReleaseMetadataConverter(github_repository={"language": "Java"})
+ self.assertEqual(
+ converter._extract_programming_languages_from_github(), ["Java"]
+ )
+
+ def test_extract_platforms_from_codemeta(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "runtimePlatform": ["Mesa", "NetLogo"],
+ }
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(
+ converter._extract_platforms_from_codemeta(), ["Mesa", "NetLogo"]
+ )
+
+ def test_extract_platforms_from_codemeta_edge_cases(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "runtimePlatform": ["Mesa", {"name": "NetLogo"}, None],
+ "releaseNotes": "From CodeMeta",
+ }
+ converter = ReleaseMetadataConverter(
+ codemeta=codemeta,
+ github_repository={"license": {"spdx_id": "MIT"}, "language": "Python"},
+ github_release={"body": "From GitHub"},
+ )
+ # Mixed structured runtimePlatform input is not supported, but conversion should remain stable.
+ self.assertIsNone(converter._extract_platforms_from_codemeta())
+ result = converter.convert()
+ self.assertIsNone(result["platforms"])
+ self.assertEqual(result["release_notes"], "From GitHub")
+ self.assertEqual(result["license_spdx_id"], "MIT")
+ self.assertEqual(result["programming_languages"], ["Python"])
+
+ def test_extract_release_notes_from_codemeta_list_edge_cases(self):
+ converter = ReleaseMetadataConverter(
+ codemeta={**self.minimal_codemeta, "releaseNotes": ["First note", "Second"]}
+ )
+ self.assertEqual(converter._extract_release_notes_from_codemeta(), "First note")
+
+ converter = ReleaseMetadataConverter(
+ codemeta={**self.minimal_codemeta, "releaseNotes": [{"text": "bad"}]}
+ )
+ self.assertIsNone(converter._extract_release_notes_from_codemeta())
+
+ def test_convert_priority(self):
+ # test the convert() method's priority logic
+ codemeta = {
+ **self.minimal_codemeta,
+ "releaseNotes": "From CodeMeta",
+ "programmingLanguage": ["Python"],
+ }
+ github_release = {"body": "From GitHub"}
+ github_repository = {"license": {"spdx_id": "MIT"}, "language": "Java"}
+ cff = {**self.minimal_cff, "license": LicenseEnum.GPL_3_0_only}
+
+ converter = ReleaseMetadataConverter(
+ codemeta=codemeta,
+ cff=cff,
+ github_repository=github_repository,
+ github_release=github_release,
+ )
+ result = converter.convert()
+ self.assertEqual(result["license_spdx_id"], "MIT") # Github > CFF
+ self.assertEqual(result["release_notes"], "From CodeMeta") # CodeMeta > Github
+ self.assertEqual(
+ result["programming_languages"], ["Java"]
+ ) # Only pull languages from GitHub
+
+ def test_convert_fallback(self):
+ """Test the convert() method's fallback logic."""
+ github_release = {"body": "From GitHub"}
+ github_repository = {"language": "Java"}
+ cff = {**self.minimal_cff, "license": LicenseEnum.GPL_3_0_only}
+
+ converter = ReleaseMetadataConverter(
+ cff=cff,
+ github_repository=github_repository,
+ github_release=github_release,
+ )
+ result = converter.convert()
+ self.assertEqual(result["license_spdx_id"], "GPL-3.0-only")
+ self.assertEqual(result["release_notes"], "From GitHub")
+ self.assertEqual(result["programming_languages"], ["Java"])
+
+ def test_convert_no_sources(self):
+ """Test that convert returns gracefully with no sources."""
+ converter = ReleaseMetadataConverter()
+ result = converter.convert()
+ expected = {
+ "license_spdx_id": None,
+ "release_notes": None,
+ "os": "",
+ "programming_languages": None,
+ "platforms": None,
+ }
+ self.assertEqual(result, expected)
diff --git a/django/library/tests/test_views.py b/django/library/tests/test_views.py
index cf5ac1f54..749568aa6 100644
--- a/django/library/tests/test_views.py
+++ b/django/library/tests/test_views.py
@@ -18,7 +18,6 @@
ApiAccountMixin,
)
from library.forms import PeerReviewerFeedbackReviewerForm
-from library.fs import FileCategoryDirectories
from library.models import (
ProgrammingLanguage,
ReleaseLanguage,
@@ -27,6 +26,7 @@
License,
PeerReview,
)
+from library.fs import FileCategories
from library.tests.base import ReviewSetup
from .base import (
CodebaseFactory,
@@ -35,7 +35,18 @@
PeerReviewInvitationFactory,
ReleaseSetup,
)
-from ..views import CodebaseViewSet, CodebaseReleaseViewSet, PeerReviewInvitationViewSet
+from ..views import (
+ CodebaseViewSet,
+ CodebaseReleaseViewSet,
+ PeerReviewInvitationViewSet,
+ github_sync_webhook,
+)
+from allauth.socialaccount.models import SocialAccount
+import json
+import hmac
+import hashlib
+from django.test import override_settings
+from library.models import GithubIntegrationAppInstallation
import logging
@@ -331,7 +342,7 @@ def test_upload_file(self):
# Unpublished codebase release permissions
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code)
+ api.get_originals_list_url(category=FileCategories.code)
)
self.assertResponseNotFound(response)
for user, expected_status_code in [
@@ -341,7 +352,7 @@ def test_upload_file(self):
]:
self.login(user, self.user_factory.password)
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code),
+ api.get_originals_list_url(category=FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -356,7 +367,7 @@ def test_upload_file(self):
# Published codebase release permissions
self.client.logout()
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code)
+ api.get_originals_list_url(category=FileCategories.code)
)
self.assertResponsePermissionDenied(response)
for user, expected_status_code in [
@@ -366,7 +377,7 @@ def test_upload_file(self):
]:
self.login(user, self.user_factory.password)
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code),
+ api.get_originals_list_url(category=FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -380,7 +391,7 @@ def test_list_files(self):
# Unpublished codebase release permissions
response = self.client.get(
- api.get_originals_list_url(category=FileCategoryDirectories.code)
+ api.get_originals_list_url(category=FileCategories.code)
)
self.assertResponseNotFound(response)
for user, expected_status_code in [
@@ -390,7 +401,7 @@ def test_list_files(self):
]:
self.login(user, self.user_factory.password)
response = self.client.get(
- api.get_originals_list_url(FileCategoryDirectories.code),
+ api.get_originals_list_url(FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -405,7 +416,7 @@ def test_list_files(self):
# Published codebase release permissions
response = self.client.get(
- api.get_originals_list_url(FileCategoryDirectories.code)
+ api.get_originals_list_url(FileCategories.code)
)
self.assertResponsePermissionDenied(response)
for user, expected_status_code in [
@@ -415,7 +426,7 @@ def test_list_files(self):
]:
self.login(user, self.user_factory.password)
response = self.client.get(
- api.get_originals_list_url(FileCategoryDirectories.code),
+ api.get_originals_list_url(FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -430,7 +441,7 @@ def test_delete_file(self):
# Unpublished codebase release permissions
response = self.client.delete(
api.get_absolute_url(
- category=FileCategoryDirectories.code, relpath=path_to_foo
+ category=FileCategories.code, relpath=path_to_foo
)
)
self.assertResponseNotFound(response)
@@ -442,7 +453,7 @@ def test_delete_file(self):
self.login(user, self.user_factory.password)
response = self.client.delete(
api.get_absolute_url(
- category=FileCategoryDirectories.code, relpath=path_to_foo
+ category=FileCategories.code, relpath=path_to_foo
),
HTTP_ACCEPT="application/json",
)
@@ -455,7 +466,7 @@ def test_delete_file(self):
# Published codebase release permissions
response = self.client.delete(
api.get_absolute_url(
- category=FileCategoryDirectories.code, relpath=path_to_foo
+ category=FileCategories.code, relpath=path_to_foo
)
)
self.assertResponsePermissionDenied(response)
@@ -466,7 +477,7 @@ def test_delete_file(self):
]:
self.login(user, self.user_factory.password)
response = self.client.delete(
- api.get_absolute_url(FileCategoryDirectories.code, path_to_foo),
+ api.get_absolute_url(FileCategories.code, path_to_foo),
HTTP_ACCEPT="application/json",
)
self.assertEqual(response.status_code, expected_status_code, msg=repr(user))
@@ -551,8 +562,8 @@ def test_publish_codebaserelease(self):
docs_file.name = "README.md"
api = self.codebase_release.get_fs_api()
- api.add(content=code_file, category=FileCategoryDirectories.code)
- api.add(content=docs_file, category=FileCategoryDirectories.docs)
+ api.add(content=code_file, category=FileCategories.code)
+ api.add(content=docs_file, category=FileCategories.docs)
with self.assertRaises(
ValidationError, msg="Codebase has no metadata, should fail publish"
@@ -596,25 +607,27 @@ def test_publish_codebaserelease(self):
class CodebaseRenderPageTestCase(TestCase):
def setUp(self):
- user_factory = UserFactory()
- self.submitter = user_factory.create()
+ self.user_factory = UserFactory()
+ self.submitter = self.user_factory.create()
codebase_factory = CodebaseFactory(submitter=self.submitter)
self.codebase = codebase_factory.create()
+ self.codebase.create_release(
+ status=CodebaseRelease.Status.PUBLISHED, initialize=False
+ )
def test_list(self):
response = self.client.get(
reverse(
- "library:codebase-detail",
- kwargs={"identifier": self.codebase.identifier},
+ "library:codebase-list"
)
)
- self.assertTrue(response.status_code, 200)
+ self.assertEqual(response.status_code, 200)
class CodebaseReleaseRenderPageTestCase(TestCase):
def setUp(self):
- user_factory = UserFactory()
- self.submitter = user_factory.create()
+ self.user_factory = UserFactory()
+ self.submitter = self.user_factory.create()
codebase_factory = CodebaseFactory(submitter=self.submitter)
contributor_factory = ContributorFactory(user=self.submitter)
contributor = contributor_factory.create(user=self.submitter)
@@ -627,6 +640,9 @@ def setUp(self):
release_contributor_factory.create(contributor=contributor)
def test_detail(self):
+ self.client.login(
+ username=self.submitter.username, password=self.user_factory.password
+ )
response = self.client.get(
reverse(
"library:codebaserelease-detail",
@@ -636,7 +652,7 @@ def test_detail(self):
},
)
)
- self.assertTrue(response.status_code, True)
+ self.assertEqual(response.status_code, 200)
class CodebaseSearchTestCase(TestCase):
@@ -755,5 +771,177 @@ def test_cannot_update_feedback_on_complete_review(self):
self.assertFalse(form.is_valid())
+class GitHubWebhookTestCase(ResponseStatusCodesMixin, TestCase):
+ def setUp(self):
+ self.user_factory = UserFactory()
+ self.user = self.user_factory.create()
+ self.path = reverse("library:github-sync-webhook")
+ self.secret = "test-secret"
+ self.factory = RequestFactory()
+
+ def _create_signed_request(self, payload, signature=None):
+ body = json.dumps(payload).encode("utf-8")
+ return self._create_signed_raw_request(body, signature=signature)
+
+ def _create_signed_raw_request(self, body, signature=None, event="installation"):
+ if signature is None:
+ hash_object = hmac.new(
+ self.secret.encode("utf-8"), msg=body, digestmod=hashlib.sha256
+ )
+ signature = f"sha256={hash_object.hexdigest()}"
+ request = self.factory.post(
+ self.path,
+ data=body,
+ content_type="application/json",
+ HTTP_X_GITHUB_EVENT=event,
+ HTTP_X_HUB_SIGNATURE_256=signature,
+ )
+ return request
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_no_matching_social_account(self):
+ # if the github account doesn't match a social account user nothing happens
+ payload = {
+ "action": "created",
+ "installation": {"id": 12345},
+ "sender": {"id": 99999, "login": "newuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 404)
+ self.assertEqual(GithubIntegrationAppInstallation.objects.count(), 0)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_creates_installation(self):
+ # if it does, it creates a installation object for them
+ SocialAccount.objects.create(
+ user=self.user,
+ provider="github",
+ uid="54321",
+ )
+ payload = {
+ "action": "created",
+ "installation": {"id": 12345},
+ "sender": {"id": 54321, "login": "testuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(GithubIntegrationAppInstallation.objects.count(), 1)
+ installation = GithubIntegrationAppInstallation.objects.first()
+ self.assertEqual(installation.user, self.user)
+ self.assertEqual(installation.installation_id, 12345)
+ self.assertEqual(installation.github_login, "testuser")
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_updates_existing_installation(self):
+ # re-installing the app on a user acc that already has the link will update the existing installation object
+ SocialAccount.objects.create(
+ user=self.user,
+ provider="github",
+ uid="54321",
+ )
+ GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ installation_id=11111,
+ github_user_id=54321,
+ github_login="testuser",
+ )
+ payload = {
+ "action": "created",
+ "installation": {"id": 22222},
+ "sender": {"id": 54321, "login": "testuser-renamed"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(GithubIntegrationAppInstallation.objects.count(), 1)
+ installation = GithubIntegrationAppInstallation.objects.first()
+ self.assertEqual(installation.installation_id, 22222)
+ self.assertEqual(installation.github_login, "testuser-renamed")
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_bad_signature(self):
+ # requests with a bad signature should be rejected
+ payload = {"action": "created"}
+ request = self._create_signed_request(payload, signature="sha256=bad-sig")
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 403)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_malformed_installation_payload_invalid_json(self):
+ request = self._create_signed_raw_request(b"{not-json")
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 400)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_malformed_installation_payload_missing_sender(self):
+ payload = {"action": "created", "installation": {"id": 12345}}
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 400)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_malformed_installation_payload_missing_installation_id(self):
+ payload = {
+ "action": "created",
+ "installation": {},
+ "sender": {"id": 54321, "login": "testuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 400)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_missing_signature_header_returns_403(self):
+ # requests that omit the signature header entirely must be rejected
+ body = json.dumps({"action": "created"}).encode("utf-8")
+ request = self.factory.post(
+ self.path,
+ data=body,
+ content_type="application/json",
+ HTTP_X_GITHUB_EVENT="installation",
+ # deliberately no HTTP_X_HUB_SIGNATURE_256
+ )
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 403)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_deletes_installation(self):
+ # action=deleted removes the installation record matched by user (via social account uid),
+ # NOT by installation_id — the id in the payload is parsed but unused by the delete path.
+ SocialAccount.objects.create(
+ user=self.user,
+ provider="github",
+ uid="54321",
+ )
+ GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ installation_id=12345,
+ github_user_id=54321,
+ github_login="testuser",
+ )
+ # use a mismatched installation id to confirm deletion is keyed on user, not installation_id
+ payload = {
+ "action": "deleted",
+ "installation": {"id": 99999},
+ "sender": {"id": 54321, "login": "testuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 200)
+ self.assertFalse(
+ GithubIntegrationAppInstallation.objects.filter(user=self.user).exists()
+ )
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_non_installation_event_returns_202(self):
+ # unrecognised event types should be acknowledged but ignored
+ body = json.dumps({"action": "opened"}).encode("utf-8")
+ request = self._create_signed_raw_request(body, event="pull_request")
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 202)
+
+
def tearDownModule():
shutil.rmtree(settings.LIBRARY_ROOT, ignore_errors=True)
diff --git a/django/library/urls.py b/django/library/urls.py
index b4e72a705..e51f58c9e 100644
--- a/django/library/urls.py
+++ b/django/library/urls.py
@@ -14,6 +14,11 @@
router.register(
r"codebases/(?P[\w\-.]+)/releases", views.CodebaseReleaseViewSet
)
+router.register(
+ r"codebases/(?P[\w\-.]+)/git/remotes",
+ views.CodebaseGitRemoteViewSet,
+ basename="codebase-git-remotes",
+)
router.register(r"reviewers", views.PeerReviewerViewSet),
router.register(r"programming-languages", views.ProgrammingLanguageViewSet),
router.register(
@@ -47,6 +52,12 @@
)
urlpatterns = [
+ path("github-sync-webhook/", views.github_sync_webhook, name="github-sync-webhook"),
+ path(
+ "github/",
+ views.GitHubSyncOverviewView.as_view(),
+ name="github-sync-overview",
+ ),
path(
"reviews/",
TemplateView.as_view(template_name="library/review/peer-review-overview.jinja"),
diff --git a/django/library/views.py b/django/library/views.py
index 909fb323e..26888e31a 100644
--- a/django/library/views.py
+++ b/django/library/views.py
@@ -1,15 +1,23 @@
+import hashlib
+import hmac
import json
+from allauth.socialaccount.models import SocialAccount
from django.forms import Form
+from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin
from django.core.exceptions import PermissionDenied
-from django.db import transaction
+from django.db import transaction, IntegrityError
+from django.db.models import Q
from django.http import HttpResponse, Http404, HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect
from django.urls import resolve, reverse
+from django.utils import timezone
from django.utils.translation import gettext_lazy as _
+from django.utils.text import slugify
from django.views import View
-from django.views.generic import FormView
+from django.views.decorators.csrf import csrf_exempt
+from django.views.generic import FormView, TemplateView
from django.views.generic.base import RedirectView
from django.views.generic.detail import DetailView
from django.views.generic.edit import UpdateView
@@ -27,6 +35,7 @@
from rest_framework.exceptions import (
PermissionDenied as DrfPermissionDenied,
ValidationError,
+ NotFound,
)
from rest_framework.filters import OrderingFilter
from rest_framework.permissions import IsAuthenticatedOrReadOnly
@@ -45,7 +54,14 @@
HtmlNoDeleteViewSet,
)
from core.pagination import SmallResultSetPagination
-from core.serializers import RelatedMemberProfileSerializer
+from .github_integration import (
+ GitHubReleaseImporter,
+ GitHubRepoValidator,
+ get_github_installation_status,
+ list_github_releases_for_remote,
+ GitHubApi,
+ extract_semver,
+)
from .forms import (
PeerReviewerFeedbackReviewerForm,
PeerReviewInvitationReplyForm,
@@ -53,12 +69,18 @@
PeerReviewerFeedbackEditorForm,
PeerReviewFilterForm,
)
-from .fs import FileCategoryDirectories, StagingDirectories, MessageLevels
+from .fs import (
+ FileCategories,
+ StagingDirectories,
+ MessageLevels,
+)
from .models import (
Codebase,
+ CodebaseGitRemote,
CodebaseRelease,
Contributor,
CodebaseImage,
+ GithubIntegrationAppInstallation,
License,
ProgrammingLanguage,
PeerReview,
@@ -66,11 +88,20 @@
PeerReviewerFeedback,
PeerReviewInvitation,
ReviewStatus,
+ GitHubIntegrationConfiguration,
+)
+from wagtail.models import Site
+from .models import GitRefSyncState, ImportedReleaseSyncState
+from .permissions import (
+ CodebaseReleaseUnpublishedFilePermissions,
+ GitHubIntegrationPermission,
)
-from .permissions import CodebaseReleaseUnpublishedFilePermissions
from .serializers import (
+ CodebaseGitRemoteSerializer,
CodebaseSerializer,
CodebaseReleaseSerializer,
+ CodebaseReleaseWithGitRefSyncStateSerializer,
+ RelatedCodebaseSerializer,
ContributorSerializer,
DownloadRequestSerializer,
PeerReviewerSerializer,
@@ -82,9 +113,15 @@
PeerReviewFeedbackEditorSerializer,
PeerReviewEventLogSerializer,
)
+from .tasks import (
+ import_github_release_task,
+ push_all_releases_to_github,
+ build_local_git_repo,
+)
import logging
import pathlib
+from packaging.version import Version
logger = logging.getLogger(__name__)
@@ -492,6 +529,28 @@ def list(self, request, *args, **kwargs):
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
+ @action(
+ detail=False, methods=["get"], permission_classes=[permissions.IsAuthenticated]
+ )
+ def submitted_codebases(self, request, *args, **kwargs):
+ """
+ Returns a list of codebases submitted by the requesting user
+ """
+ queryset = (
+ self.get_queryset().filter(submitter=request.user).order_by("-date_created")
+ )
+ serializer = RelatedCodebaseSerializer(
+ queryset,
+ many=True,
+ context=self.get_serializer_context(),
+ )
+ return Response(serializer.data)
+
+ @action(detail=False, methods=["get"])
+ def github_installation_status(self, request, *args, **kwargs):
+ data = get_github_installation_status(request.user)
+ return Response(data=data, status=status.HTTP_200_OK)
+
def get_queryset(self):
if self.action == "list":
return self.queryset.public()
@@ -501,7 +560,8 @@ def get_queryset(self):
def perform_create(self, serializer):
super().perform_create(serializer)
codebase = serializer.instance
- return codebase.get_or_create_draft()
+ initial_version = self.request.query_params.get("initial_version")
+ return codebase.get_or_create_draft(initial_version=initial_version)
def retrieve(self, request, *args, **kwargs):
instance = self.get_object()
@@ -568,18 +628,379 @@ def get_redirect_url(self, *args, **kwargs):
return super().get_redirect_url(*args, **kwargs)
-class CodebaseImagePermission(permissions.BasePermission):
+class CanChangeCodebase(permissions.BasePermission):
def has_permission(self, request, view):
codebase = get_object_or_404(Codebase, identifier=view.kwargs["identifier"])
return request.user.has_perm("library.change_codebase", codebase)
+class CodebaseGitRemoteViewSet(
+ mixins.RetrieveModelMixin, mixins.UpdateModelMixin, viewsets.GenericViewSet
+):
+ queryset = CodebaseGitRemote.objects.all()
+ permission_classes = (CanChangeCodebase, GitHubIntegrationPermission)
+ pagination_class = None
+ serializer_class = CodebaseGitRemoteSerializer
+
+ def get_queryset(self):
+ resolved = resolve(self.request.path)
+ identifier = resolved.kwargs["identifier"]
+ return self.queryset.filter(codebase__identifier=identifier).prefetch_related(
+ "git_ref_sync_states", "imported_sync_states"
+ )
+
+ def get_codebase(self):
+ try:
+ return Codebase.objects.get(identifier=self.kwargs.get("identifier"))
+ except Codebase.DoesNotExist:
+ raise NotFound("Codebase not found.")
+
+ def _forward_integrity_error(self, e: IntegrityError):
+ msg = str(e)
+ if "single_active_remote" in msg:
+ raise ValidationError(
+ "There can only be one active repository for a codebase at a time."
+ )
+ else:
+ raise ValidationError(
+ "This repository is already connected to another model."
+ )
+
+ def update(self, request, *args, **kwargs):
+ try:
+ return super().update(request, *args, **kwargs)
+ except IntegrityError as e:
+ self._forward_integrity_error(e)
+
+ def list(self, request, *args, **kwargs):
+ queryset = self.get_queryset()
+ if request.accepted_renderer.format == "html":
+ context = {
+ "codebase": self.get_codebase(),
+ "remotes": queryset,
+ }
+ return Response(
+ context,
+ template_name="library/codebases/git.jinja",
+ )
+ serializer = self.get_serializer(queryset, many=True)
+ return Response(serializer.data)
+
+ @action(detail=False, methods=["post"])
+ def push_all(self, request, *args, **kwargs):
+ """Manually push all releases to GitHub for the active remote.
+
+ Immediately marks each release's GitRefSyncState as RUNNING for UI feedback,
+ then enqueues an async task to perform the push and create GitHub releases.
+ """
+ codebase = self.get_codebase()
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ raise ValidationError("Active remote not found")
+
+ # prevent concurrent push jobs for the same codebase
+ if GitRefSyncState.objects.filter(
+ Q(release__codebase=codebase) | Q(codebase=codebase),
+ status=GitRefSyncState.Status.RUNNING,
+ ).exists():
+ raise ValidationError(
+ "Push already in progress for this codebase. Please wait."
+ )
+
+ # start all push jobs for this remote (set remotes and mark running)
+ started = GitRefSyncState.start_all_push_jobs_for_codebase(
+ codebase, active_remote
+ )
+ if not started:
+ return Response(status=status.HTTP_204_NO_CONTENT)
+ # enqueue task to carry out all the push jobs we just started
+ push_all_releases_to_github(codebase.id, active_remote.id)
+ return Response(status=status.HTTP_202_ACCEPTED, data="Push started.")
+
+ @action(detail=False, methods=["get"])
+ def active_remote(self, request, *args, **kwargs):
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if active_remote:
+ serializer = self.get_serializer(active_remote)
+ return Response(data=serializer.data, status=status.HTTP_200_OK)
+ return Response(data=None, status=status.HTTP_200_OK)
+
+ @action(detail=False, methods=["get"])
+ def local_releases(self, request, *args, **kwargs):
+ """Return all local releases for this codebase with git ref sync states."""
+ codebase = self.get_codebase()
+ qs = CodebaseRelease.objects.filter(codebase=codebase).select_related(
+ "codebase", "git_ref_sync_state"
+ )
+ releases = list(qs)
+ releases.sort(key=lambda r: Version(r.version_number), reverse=True)
+ serializer = CodebaseReleaseWithGitRefSyncStateSerializer(
+ releases, many=True, context=self.get_serializer_context()
+ )
+ return Response(serializer.data)
+
+ @action(detail=False, methods=["get"])
+ def github_releases(self, request, *args, **kwargs):
+ """fetch releases from the connected GitHub repository for this codebase
+
+ requires an active remote. returns minimal release info suitable for client display
+ and import selection.
+ """
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ return Response([], status=status.HTTP_200_OK)
+ try:
+ releases = list_github_releases_for_remote(active_remote)
+ return Response(releases, status=status.HTTP_200_OK)
+ except Exception as e:
+ logger.exception("Failed to fetch GitHub releases: %s", e)
+ raise ValidationError(
+ "Unable to fetch GitHub releases for the active remote"
+ )
+
+ @action(detail=False, methods=["post"])
+ def import_github_release(self, request, *args, **kwargs):
+ """Start an async import for a specific GitHub release id for the active remote.
+
+ body params:
+ - github_release_id: str | int (required)
+ - custom_version: str (optional) when the release tag/name has no semver
+ """
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ raise ValidationError("Active remote not found")
+ github_release_id = request.data.get("github_release_id")
+ if not github_release_id:
+ raise ValidationError("'github_release_id' is required")
+ custom_version = request.data.get("custom_version")
+ if custom_version is not None:
+ if not isinstance(custom_version, str):
+ raise ValidationError(
+ "'custom_version' must be a semantic version like X.Y.Z"
+ )
+ candidate = custom_version.strip()
+ parsed_version = extract_semver(candidate)
+ is_exact_semver = parsed_version and (
+ candidate == parsed_version or candidate == f"v{parsed_version}"
+ )
+ if not is_exact_semver:
+ raise ValidationError(
+ "'custom_version' must be a semantic version like X.Y.Z"
+ )
+ custom_version = parsed_version
+ # require existing sync state
+ sync_state = ImportedReleaseSyncState.objects.filter(
+ remote=active_remote, github_release_id=str(github_release_id)
+ ).first()
+ if not sync_state:
+ raise ValidationError(
+ "Import not prepared. refresh GitHub releases and try again."
+ )
+ if sync_state.status == ImportedReleaseSyncState.Status.RUNNING:
+ raise ValidationError("GitHub release import is already in progress")
+ # mark as RUNNING and dispatch import task
+ sync_state.mark_running()
+ import_github_release_task(
+ active_remote.codebase_id,
+ active_remote.id,
+ str(github_release_id),
+ custom_version,
+ )
+ return Response(status=status.HTTP_202_ACCEPTED, data="Import started.")
+
+ @action(detail=False, methods=["get"])
+ def submitter_installation_status(self, request, *args, **kwargs):
+ codebase = self.get_codebase()
+ submitter = codebase.submitter
+ data = get_github_installation_status(submitter)
+ return Response(data=data, status=status.HTTP_200_OK)
+
+ @action(detail=False, methods=["post"])
+ def setup_user_github_remote(self, request, *args, **kwargs):
+ codebase = self.get_codebase()
+ installation = codebase.submitter.github_integration_app_installation
+ if not installation:
+ raise ValidationError(
+ "Installation of the Github integration app is required"
+ )
+
+ is_preexisting = bool(request.data.get("is_preexisting", False))
+ if not codebase.live and not is_preexisting:
+ raise ValidationError("This model does not have any published releases")
+
+ repo_name = request.data.get("repo_name")
+ if not repo_name:
+ raise ValidationError("Repository name is required")
+
+ validator = GitHubRepoValidator(repo_name)
+ try:
+ validator.validate_format()
+ repo_html_url = validator.get_url_for_connectable_user_repo(
+ installation, is_preexisting
+ )
+ except ValueError as e:
+ error_message = e.args[0] if e.args else "Invalid repository configuration"
+ raise ValidationError(error_message)
+
+ # swap out a new active remote for the old one
+ queryset = self.get_queryset()
+ current_remote = queryset.filter(is_active=True).first()
+ try:
+ with transaction.atomic():
+ if current_remote:
+ current_remote.is_active = False
+ current_remote.save(update_fields=["is_active", "last_modified"])
+ new_remote = codebase.create_remote(
+ owner=installation.github_login,
+ repo_name=repo_name,
+ is_user_repo=True,
+ is_preexisting=is_preexisting,
+ is_active=True,
+ url=repo_html_url,
+ )
+ # check if the refs were previously bound to a different remote and reassign
+ # this handles both: switching from an active remote, and reconnecting after
+ # a manual disconnect (where the refs still point to the old remote)
+ previous_remote = GitRefSyncState.get_last_remote_for_codebase(codebase)
+ if previous_remote and previous_remote.id != new_remote.id:
+ GitRefSyncState.reassign_remotes_for_codebase(
+ codebase=codebase, remote=new_remote
+ )
+ except IntegrityError as e:
+ self._forward_integrity_error(e)
+ return Response(
+ status=status.HTTP_200_OK,
+ data="Repository connection successful.",
+ )
+
+ @action(detail=False, methods=["post"])
+ def disconnect_remote(self, request, *args, **kwargs):
+ """Deactivate the current active remote without reassigning refs.
+
+ This allows users to disconnect without losing push state, so they can
+ reconnect to the same remote later if desired.
+ """
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ raise ValidationError("No active remote to disconnect")
+
+ active_remote.is_active = False
+ active_remote.save(update_fields=["is_active", "last_modified"])
+ return Response(
+ status=status.HTTP_200_OK,
+ data="Repository disconnected.",
+ )
+
+ @action(detail=False, methods=["post"])
+ def build_local_repo(self, request, *args, **kwargs):
+ """Build or update the local git mirror for this codebase without pushing."""
+ codebase = self.get_codebase()
+ build_local_git_repo(codebase.id)
+ return Response(
+ status=status.HTTP_202_ACCEPTED, data="Local repository build started."
+ )
+
+
+class GitHubSyncOverviewView(TemplateView):
+ """View for GitHub integration overview page with permission check."""
+
+ template_name = "library/github_overview.jinja"
+
+ def dispatch(self, request, *args, **kwargs):
+ try:
+ site = Site.objects.get(is_default_site=True)
+ config = GitHubIntegrationConfiguration.for_site(site)
+ except (Site.DoesNotExist, GitHubIntegrationConfiguration.DoesNotExist):
+ raise PermissionDenied("GitHub integration is not available")
+
+ if not config.can_use_github_integration(request.user):
+ raise PermissionDenied("GitHub integration is not available")
+ return super().dispatch(request, *args, **kwargs)
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+ context["github_model_library_org_name"] = (
+ settings.GITHUB_MODEL_LIBRARY_ORG_NAME
+ )
+ return context
+
+
+@csrf_exempt
+def github_sync_webhook(request):
+ """
+ Handle GitHub app webhook events:
+ - new installations (match to connected socialaccount and save the installation record)
+
+ for local testing, use smee.io or similar to forward webhooks:
+ https://docs.github.com/en/webhooks/using-webhooks/handling-webhook-deliveries#setup
+ """
+ # verify the request signature
+ signature_header = request.headers.get("X-Hub-Signature-256")
+ if not signature_header:
+ return HttpResponse("Missing signature header", status=403)
+ secret = settings.GITHUB_INTEGRATION_APP_WEBHOOK_SECRET
+ hash_object = hmac.new(
+ secret.encode("utf-8"),
+ msg=request.body,
+ digestmod=hashlib.sha256,
+ )
+ expected_signature = f"sha256={hash_object.hexdigest()}"
+ if not hmac.compare_digest(expected_signature, signature_header):
+ return HttpResponse("Invalid signature", status=403)
+
+ # parse the event
+ event = request.headers.get("X-GitHub-Event")
+ if event == "installation":
+ try:
+ payload = json.loads(request.body)
+ logger.debug("received installation webhook %s", payload)
+ action = payload["action"]
+ sender = payload["sender"]
+ installation_id = payload["installation"]["id"]
+ sender_login = sender["login"]
+ except (json.JSONDecodeError, KeyError, TypeError):
+ return HttpResponse("Malformed payload", status=400)
+ # match based on the uid
+ uid = sender["id"]
+ social_account = SocialAccount.objects.filter(
+ provider="github", uid=str(uid)
+ ).first()
+ if not social_account:
+ logger.warning(
+ "No matching social account found for installation event: %s", payload
+ )
+ return HttpResponse(status=404)
+ if action == "deleted":
+ installation = GithubIntegrationAppInstallation.objects.filter(
+ user=social_account.user,
+ ).delete()
+ return HttpResponse("OK", status=200)
+ else:
+ installation = GithubIntegrationAppInstallation.objects.update_or_create(
+ user=social_account.user,
+ defaults={
+ "github_user_id": uid,
+ "github_login": sender_login,
+ "installation_id": installation_id,
+ },
+ )
+ logger.debug("Installation record updated/created: %s", installation)
+ return HttpResponse("OK", status=200)
+
+ return HttpResponse(status=202)
+
+
class CodebaseImageViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
lookup_field = "codebaseimage_id"
lookup_value_regex = r"\d+"
queryset = CodebaseImage.objects.all()
serializer_class = CodebaseImageSerializer
- permission_classes = (CodebaseImagePermission,)
+ permission_classes = (CanChangeCodebase,)
def get_queryset(self):
resolved = resolve(self.request.path)
@@ -885,7 +1306,7 @@ def notify_reviewers_of_changes(self, request, **kwargs):
@transaction.atomic
def request_peer_review(self, request, identifier, version_number):
"""
- If a peer review is requestable (publishable, not reviewed, no related review exists):
+ If a peer review is requestable (publishable, not reviewed, no related review exists, and not imported+published):
- Create a new "under review" draft release if the release from which the request was made is published
- Otherwise, update the existing draft release to be "under review"
- Create a new peer review object and send an email to the author
@@ -895,25 +1316,40 @@ def request_peer_review(self, request, identifier, version_number):
codebase__identifier=identifier,
version_number=version_number,
)
+ if codebase_release.is_imported and codebase_release.is_published:
+ raise ValidationError(
+ "Cannot request a peer review for an imported release that has already been published. As a workaround, you can make another release on GitHub and request a peer review for that."
+ )
codebase_release.validate_publishable()
existing_review = PeerReview.get_codebase_latest_active_review(
codebase_release.codebase
)
+ # if there is an existing active review, simply redirect to that release
if existing_review:
review = existing_review
review_release = existing_review.codebase_release
created = False
+ # if not create a new review
else:
- if codebase_release.is_draft:
- codebase_release.status = CodebaseRelease.Status.UNDER_REVIEW
- codebase_release.save(update_fields=["status"])
- review_release = codebase_release
- else:
+ # first, check if this release status is already under or completed review
+ # if so, something went wrong so we'll
+ if codebase_release.is_under_review or codebase_release.is_review_complete:
+ raise ValidationError(
+ "Cannot re-request a review on a release that has already completed or is undergoing review."
+ )
+ # if the release is published, make a new draft copy
+ elif codebase_release.is_published:
review_release = (
codebase_release.codebase.create_review_draft_from_release(
codebase_release
)
)
+ # if the release is a draft/unpublished, change the status
+ else:
+ codebase_release.status = CodebaseRelease.Status.UNDER_REVIEW
+ codebase_release.save(update_fields=["status"])
+ review_release = codebase_release
+
review = PeerReview.objects.create(
codebase_release=review_release,
submitter=request.user.member_profile,
@@ -1027,7 +1463,7 @@ def get_url_matcher(cls):
r"codebases/(?P[\w\-.]+)",
r"/releases/(?P\d+\.\d+\.\d+)",
r"/files/{}/(?P{})".format(
- cls.stage.name, "|".join(c.name for c in FileCategoryDirectories)
+ cls.stage.name, "|".join(c.name for c in FileCategories)
),
]
)
@@ -1038,20 +1474,20 @@ def get_queryset(self):
queryset = self.queryset.filter(codebase__identifier=identifier)
return queryset.accessible(user=self.request.user)
- def get_category(self) -> FileCategoryDirectories:
+ def get_list_url(self, api):
+ raise NotImplementedError
+
+ def get_category(self) -> FileCategories:
category = self.get_parser_context(self.request)["kwargs"]["category"]
try:
- return FileCategoryDirectories[category]
+ return FileCategories[category]
except KeyError:
raise ValidationError(
- "Target folder name {} invalid. Must be one of {}".format(
- category, list(d.name for d in FileCategoryDirectories)
+ "Target category name {} invalid. Must be one of {}".format(
+ category, list(d.name for d in FileCategories)
)
)
- def get_list_url(self, api):
- raise NotImplementedError
-
def list(self, request, *args, **kwargs):
codebase_release = self.get_object()
api = codebase_release.get_fs_api()
@@ -1090,6 +1526,35 @@ class CodebaseReleaseFilesSipViewSet(BaseCodebaseReleaseFilesViewSet):
def get_list_url(self, api):
return api.get_sip_list_url
+ @action(detail=False, methods=["post"])
+ def update_category(self, request, **kwargs):
+ """update a file's category, currently only for imported releases
+
+ Note: the category given in the request data is the new category for the file
+ and the category in the URL is the current category of the file (or anything,
+ it is ignored here)
+ """
+ codebase_release = self.get_object()
+ if not codebase_release.is_imported:
+ raise ValidationError("Cannot update file category on non-imported release")
+ fs_api = codebase_release.get_fs_api()
+ file_path = request.data.get("path")
+ new_category_str = request.data.get("category")
+ if not file_path or not new_category_str:
+ raise ValidationError("Both a file 'path' and 'category' are required")
+ try:
+ new_category = FileCategories[new_category_str]
+ except KeyError:
+ raise ValidationError(
+ f"Target category name {new_category_str} invalid. Must be one of {[c.name for c in FileCategories]}"
+ )
+ try:
+ fs_api.manifest.update_file_category(file_path, new_category)
+ except ValueError as e:
+ error_message = e.args[0] if e.args else "Unable to update file category"
+ raise ValidationError(error_message)
+ return Response(status=status.HTTP_200_OK)
+
class CodebaseReleaseFilesOriginalsViewSet(BaseCodebaseReleaseFilesViewSet):
stage = StagingDirectories.originals
diff --git a/django/requirements.txt b/django/requirements.txt
index 09d4f6690..16ba024ca 100644
--- a/django/requirements.txt
+++ b/django/requirements.txt
@@ -27,6 +27,7 @@ Django==5.2.11
drf-spectacular==0.29.0
elasticsearch-dsl>=7.0.0,<8.0.0
elasticsearch>=7.0.0,<8.0.0
+gitpython==3.1.50
html2text>=2016.9.19
huey==2.6.0
jinja2==3.1.6
@@ -37,6 +38,7 @@ numpy==1.26.4
pandas==2.2.2
psycopg2-binary==2.9.11
pyjwt[crypto]>=2.4.0,<3.0.0
+PyGithub==2.5.0
pytz==2025.2
pyyaml>=6.0.1
# used for institution -> affiliation data migration
@@ -56,6 +58,5 @@ wagtail-modeladmin==2.2.0
invoke
ipython
pgcli
-# replace with pypi scicodes install eventually
git+https://github.com/comses/codemeticulous.git
sqlparse>=0.5.4 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/frontend/src/apps/codebase_edit.ts b/frontend/src/apps/codebase_edit.ts
index 021a76880..c31bb185c 100644
--- a/frontend/src/apps/codebase_edit.ts
+++ b/frontend/src/apps/codebase_edit.ts
@@ -4,5 +4,5 @@ import { createApp } from "vue";
import CodebaseEditForm from "@/components/CodebaseEditForm.vue";
import { extractDataParams } from "@/util";
-const props = extractDataParams("codebase-form", ["identifier"]);
+const props = extractDataParams("codebase-form", ["identifier", "canUseGithubIntegration"]);
createApp(CodebaseEditForm, props).mount("#codebase-form");
diff --git a/frontend/src/apps/github_config.ts b/frontend/src/apps/github_config.ts
new file mode 100644
index 000000000..3b4d909be
--- /dev/null
+++ b/frontend/src/apps/github_config.ts
@@ -0,0 +1,12 @@
+import "vite/modulepreload-polyfill";
+
+import { createApp } from "vue";
+import { extractDataParams } from "@/util";
+import GitHubIntegrationConfiguration from "@/components/GitHubIntegrationConfiguration.vue";
+
+const props = extractDataParams("github-config", [
+ "codebaseIdentifier",
+ "defaultRepoName",
+ "isCodebaseLive",
+]);
+createApp(GitHubIntegrationConfiguration, props).mount("#github-config");
diff --git a/frontend/src/apps/github_overview.ts b/frontend/src/apps/github_overview.ts
new file mode 100644
index 000000000..0293c5dc8
--- /dev/null
+++ b/frontend/src/apps/github_overview.ts
@@ -0,0 +1,6 @@
+import "vite/modulepreload-polyfill";
+
+import { createApp } from "vue";
+import GitHubIntegrationOverview from "@/components/GitHubIntegrationOverview.vue";
+
+createApp(GitHubIntegrationOverview).mount("#github-overview");
diff --git a/frontend/src/apps/release_editor.ts b/frontend/src/apps/release_editor.ts
index 8b479a2a9..28cd4585c 100644
--- a/frontend/src/apps/release_editor.ts
+++ b/frontend/src/apps/release_editor.ts
@@ -6,6 +6,7 @@ import { createRouter, createWebHashHistory } from "vue-router";
import App from "@/components/releaseEditor/App.vue";
import MetadataFormPage from "@/components/releaseEditor/MetadataFormPage.vue";
import UploadFormPage from "@/components/releaseEditor/UploadFormPage.vue";
+import ImportedArchivePage from "@/components/releaseEditor/ImportedArchivePage.vue";
import ContributorsPage from "@/components/releaseEditor/ContributorsPage.vue";
import { extractDataParams } from "@/util";
@@ -15,6 +16,7 @@ const props = extractDataParams("release-editor", [
"reviewStatus",
"isLive",
"canEditOriginals",
+ "isImported",
]);
// check if ?publish or ?upload-image is in the url, set prop, and clear from the url
@@ -26,14 +28,21 @@ window.history.replaceState({}, "", window.location.pathname + window.location.h
const app = createApp(App, props);
const pinia = createPinia();
+const filesRouteName = props.isImported ? "package" : "upload";
+const filesRouteComponent = props.isImported ? ImportedArchivePage : UploadFormPage;
+
const router = createRouter({
history: createWebHashHistory(),
routes: [
- // only include upload route when original files are editable
- { path: "/", redirect: { name: props.canEditOriginals ? "upload" : "metadata" } },
- ...(props.canEditOriginals
- ? [{ path: "/upload", component: UploadFormPage, name: "upload" }]
- : []),
+ // only default to upload route when original files are editable
+ {
+ path: "/",
+ redirect: {
+ name: props.canEditOriginals ? filesRouteName : "metadata",
+ },
+ },
+ // use the imported archive page for imported releases
+ { path: `/${filesRouteName}`, component: filesRouteComponent, name: filesRouteName },
{ path: "/metadata", component: MetadataFormPage, name: "metadata" },
{ path: "/contributors", component: ContributorsPage, name: "contributors" },
],
diff --git a/frontend/src/components/CodebaseEditForm.vue b/frontend/src/components/CodebaseEditForm.vue
index 583b2e6e3..644f298f2 100644
--- a/frontend/src/components/CodebaseEditForm.vue
+++ b/frontend/src/components/CodebaseEditForm.vue
@@ -48,27 +48,36 @@
help="Add tags to categorize your model and make it more discoverable. Press enter after entering each tag."
/>
+ Manage this connection by importing releases from GitHub or pushing releases to GitHub
+ below. Return here in the future by pressing
+ manage in the GitHub panel on the model page.
+
+
+
+
+
+
+ This will deactivate the connection to
+ {{ activeRemote?.owner }}/{{ activeRemote?.repoName }}.
+
+
+ If you reconnect to the same repository later, your push history will be preserved.
+ However, if you connect to a different repository, push history will be reset.
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/GitHubIntegrationOverview.vue b/frontend/src/components/GitHubIntegrationOverview.vue
new file mode 100644
index 000000000..4b67fcb29
--- /dev/null
+++ b/frontend/src/components/GitHubIntegrationOverview.vue
@@ -0,0 +1,95 @@
+
+
+
+
+ Your submitted models
+ Submit a new model
+
+
+
+ Loading...
+
+
+ You have not submitted any models yet.
+
+
+
+
+
+
+
+
+
+
+
+
Peer reviews must be requested on
unpublished model releases so that you can continue to revise your submission
- based on reviewer guidance. If you have already published your model
- a new draft release will be created as an exact copy of your most recent
- published release.
+ based on reviewer guidance.
+
+ -
+ If you have already published this release,
+ a new draft will be created as an exact copy in order to allow for changes.
+
+ -
+ If this release was imported from GitHub and published, you will not be able to request a
+ review, and will need to create a new release on GitHub for peer review to take place on.
+
+
Once a model release passes peer review it will remain private until you choose to publish it,
at which point you may set a version number. Please note that
diff --git a/frontend/src/components/form/HoneypotField.vue b/frontend/src/components/form/HoneypotField.vue
index cd07c44a2..e07d62a1b 100644
--- a/frontend/src/components/form/HoneypotField.vue
+++ b/frontend/src/components/form/HoneypotField.vue
@@ -36,7 +36,7 @@ import FieldHelp from "@/components/form/FieldHelp.vue";
export interface HoneypotFieldProps {
name?: string;
- show: boolean;
+ show?: boolean;
}
const props = withDefaults(defineProps(), {
diff --git a/frontend/src/components/githubIntegration/AccordionStep.vue b/frontend/src/components/githubIntegration/AccordionStep.vue
new file mode 100644
index 000000000..beeae37f7
--- /dev/null
+++ b/frontend/src/components/githubIntegration/AccordionStep.vue
@@ -0,0 +1,48 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Connect GitHub Account
+
+ Signed in as
+
+
+ {{ githubAccount.username }}
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Connect your GitHub account with your CoMSES Net profile
+
+
+
+ Connect GitHub
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/ConnectRepositoryStep.vue b/frontend/src/components/githubIntegration/ConnectRepositoryStep.vue
new file mode 100644
index 000000000..bac632dd6
--- /dev/null
+++ b/frontend/src/components/githubIntegration/ConnectRepositoryStep.vue
@@ -0,0 +1,93 @@
+
+
+
+ Connect Repository
+
+ Connected to
+
+
+ {{ activeRemote!.owner }}/{{ activeRemote!.repoName }}
+
+
+
+
+
+
+
+
+
+
+
+
+ Make sure the CoMSES Integration GitHub App has access to the repository you want to
+ connect.
+ Manage permissions
+
+
+
+ Enter the name of your repository to connect it
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/GitHubReleaseItem.vue b/frontend/src/components/githubIntegration/GitHubReleaseItem.vue
new file mode 100644
index 000000000..6cd97d8d2
--- /dev/null
+++ b/frontend/src/components/githubIntegration/GitHubReleaseItem.vue
@@ -0,0 +1,34 @@
+
+
+
+
+
+ Released {{ (release.publishedAt || release.createdAt)?.split("T")[0] }}
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/InstallGitHubAppStep.vue b/frontend/src/components/githubIntegration/InstallGitHubAppStep.vue
new file mode 100644
index 000000000..096f6ea50
--- /dev/null
+++ b/frontend/src/components/githubIntegration/InstallGitHubAppStep.vue
@@ -0,0 +1,70 @@
+
+
+
+ Install CoMSES Integration GitHub App
+ CoMSES Integration GitHub App installed
+
+
+
+
+
+
+
+
+
+
+
+
+ This gives our service read and write access to repositories. Make sure to select the
+ individual repository(s) you want to connect, or allow access to all repositories (not
+ recommended).
+
+
+
+
+
+ Install App
+
+
+ Installed but not recognized? Try refreshing the page or
+
+ uninstalling and reinstalling the app .
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/LocalReleaseItem.vue b/frontend/src/components/githubIntegration/LocalReleaseItem.vue
new file mode 100644
index 000000000..b29ee77a7
--- /dev/null
+++ b/frontend/src/components/githubIntegration/LocalReleaseItem.vue
@@ -0,0 +1,80 @@
+
+
+
+
+
+
+
+
+
+ Verify import and notify reviewers
+
+ Verify import
+
+
+
+ Only published releases are included in git repos
+
+
+ Published {{ release.lastPublishedOn }}
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/ReleaseManagementSection.vue b/frontend/src/components/githubIntegration/ReleaseManagementSection.vue
new file mode 100644
index 000000000..e334e7e87
--- /dev/null
+++ b/frontend/src/components/githubIntegration/ReleaseManagementSection.vue
@@ -0,0 +1,406 @@
+
+
+
+
+
+
+
+
+
+ 
+ Model Library releases
+
+
+
+
+
+
+
+
+
+
+ GitHub releases
+
+
+ No releases found. You must
+
+ create a release on GitHub
+ in order to import to the model library.
+
+
+
+
+
+
+
+
+ Fetching releases...
+
+
+
+ No releases available
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/ReleaseSyncStatusCell.vue b/frontend/src/components/githubIntegration/ReleaseSyncStatusCell.vue
new file mode 100644
index 000000000..096459002
--- /dev/null
+++ b/frontend/src/components/githubIntegration/ReleaseSyncStatusCell.vue
@@ -0,0 +1,210 @@
+
+
+
+
+
+
+
+
+
+
+
+ {{ badgeText }}
+
+
+
+
+
+ This release does not conform to semantic versioning. Please specify a version number.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/SelectSyncTypeStep.vue b/frontend/src/components/githubIntegration/SelectSyncTypeStep.vue
new file mode 100644
index 000000000..5309ee059
--- /dev/null
+++ b/frontend/src/components/githubIntegration/SelectSyncTypeStep.vue
@@ -0,0 +1,124 @@
+
+
+
+ Select Sync Type
+
+ {{
+ selectedSyncType === "existing"
+ ? "Syncing with existing repository"
+ : "Syncing with new repository"
+ }}
+
+
+
+
+
+
+
+
+
+
+ Connect to a repository you already have on GitHub, allowing you to
+ import releases made there to the CoMSES Model Library.
+
+
+
+
+
+
+
+
+ You will create a new GitHub repository, then a git repo will be automatically
+ constructed from your releases published here, which you can
+ push. This is an easy way to start using GitHub and you'll be able to
+ import any future releases created there.
+
+
+
+
+
+
+
+
+
+
+
+ You'll need to create an empty repository on GitHub first. After you connect it, the
+ history will be built and you'll be able to push it.
+
+
+
+
+
+ Name the repository anything you want, but make sure it is public and
+ NOT initialized
+ with anything like a README, .gitignore, or license files.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/releaseEditor/App.vue b/frontend/src/components/releaseEditor/App.vue
index 90b161236..dc1b290d9 100644
--- a/frontend/src/components/releaseEditor/App.vue
+++ b/frontend/src/components/releaseEditor/App.vue
@@ -6,9 +6,9 @@
- Private
+ Private
{{ store.release.codebase.title }}
@@ -25,7 +25,7 @@
Review Status:
{{ reviewStatus }}
-
-
+ Below is the current filesystem layout of the archival package as it was imported from GitHub.
+ Note that metadata files (CITATION.cff, codemeta.json) are
+ automatically updated to contain the most accurate information about the model release and
+ reflect its location in the CoMSES Model Library.
+
+
+ Please review the file tree to ensure that all necessary files are present and re-categorize
+ files as necessary. At least one code and documentation file is required for publishing.
+
+
+ If you do need to add, remove, or update any files, you can do so by
+ updating the source GitHub release
+ to point at a new tag with the desired changes, which will trigger a re-import of the release.
+ This can only be done if the release is unpublished or under review, once it is published the
+ file package is locked.
+
+
+ Current Archival Package Filesystem Layout
+ Loading download preview...
+
+ {{ serverErrors.join(", ") }}
+
+
+
GitHub Integration Overview{% if github_config.is_beta %} Beta{% endif %}
+{% endwith %} +{% endblock %} + +{% block content %} +{% with github_config=settings("library.GitHubIntegrationConfiguration", use_default_site=True) %} +{% macro icon_bullets(items) -%} +-
+ {%- for item in items %}
+ {% set icon = item.icon or "fas fa-chevron-right" %}
+
-
+
+ {{ item.text.rendered|safe }}+
+ {%- endfor %}
+
+ {% if video.title %}
+
+{%- endfor %}
+{%- endmacro %}
+
+{% macro faq_accordion(entries, accordion_id="github-faq-accordion") -%}
+{{ video.title }}
+ {% endif %} + + {% if video.caption and video.caption.raw %} +{{ video.caption.rendered|safe }}
+ {% endif %}
+
+ {% for entry in entries %}
+ {% set entry_id = accordion_id ~ "_entry_" ~ loop.index %}
+ {% set heading_id = accordion_id ~ "_heading_" ~ loop.index %}
+
+{%- endmacro %}
+
+
+
+ {% endfor %}
+
+
+ + +
+
+
+
+ {{ entry.answer.rendered|safe }}
+
+
+
+{% endwith %}
+{% endblock %}
+
+{% block js %}
+{{ vite_asset("apps/github_overview.ts") }}
+{% endblock %}
\ No newline at end of file
diff --git a/django/library/jinja2/library/includes/publish_model_help.jinja b/django/library/jinja2/library/includes/publish_model_help.jinja
index 24dc87e18..7fdf3900e 100644
--- a/django/library/jinja2/library/includes/publish_model_help.jinja
+++ b/django/library/jinja2/library/includes/publish_model_help.jinja
@@ -1,26 +1,47 @@
-
+
+
+
+
+
+
+
+
+ {% if user.is_authenticated %}
+
+ {% else %}
+ + The CoMSES Model Library GitHub integration + allows you to transfer releases of your model to or from a GitHub + repository by connecting an existing repository or by having one created for you. +
++ Get started by pressing + + Connect a repository + + for one of your models listed below or on the model's + page. You can also import directly from an existing GitHub repository when submitting a new model by selecting + Import model from GitHub + on the new model form. + Once you have a connected repository, it will be linked on the model's page and + visible to others. +
+
+ You must be signed
+ in to manage the GitHub integration for your models.
+
+ {% endif %}
+
+
+ Learn more about how it works
+
+
+
+
+
+
+ Import model code from GitHub
+Have an existing GitHub repository for your model? Connect + it to import releases straight into the CoMSES Model Library.
+
+
+ Read more
+
+
+
+
+
+
+
+
+ Set up a new GitHub repository
+Don't have a repository yet? We'll build a git repository + from your existing releases and let you push it to your own GitHub account.
+
+
+ Read more
+
+
+
+
+
+
+
+
+ Have questions? Concerns?
+Check out the list of frequently ask questions. If you + don't find an answer, we are happy to help, just contact us.
+
+
+ FAQs
+
+
+ +
+
+
+
+ Import models from an existing GitHub repository
+
+
+
+ {% if github_config %}
+ {{ tutorial_videos(github_config.existing_repo_tutorial_videos.all()) }}
+ {% endif %}
+
+ {% if github_config %}
+ {{ icon_bullets(github_config.existing_repo_info_items.all()) }}
+ {% endif %}
+ +
+
+
+
+ Set up a new GitHub repository for your submitted model
+
+
+
+ {% if github_config %}
+ {{ tutorial_videos(github_config.new_repo_tutorial_videos.all()) }}
+ {% endif %}
+
+ {% if github_config %}
+ {{ icon_bullets(github_config.new_repo_info_items.all()) }}
+ {% endif %}
+ +
+
+
+
+ Frequently asked questions
+ {% if github_config and github_config.faq_entries.exists() %} + {{ faq_accordion(github_config.faq_entries.all()) }} + {% endif %} +
-
- About the CoMSES Model Library
-
- more info
+
- About the CoMSES Model Library
-
- more info
+
+
+
+ About the Model Library
-
-
-
-
- Our mission is to help computational modelers develop, document, and share their computational models in accordance with community standards and good open science and software engineering practices. Model authors can publish their model source code in the Computational Model Library with narrative documentation as well as metadata that supports open science and emerging norms that facilitate software citation, computational reproducibility / frictionless reuse, and interoperability. Model authors can also request private peer review of their computational models. Models that pass peer review receive a DOI once published.
-
-
- All users of models published in the library must cite model authors when they use and benefit from their code.
-
+
+ Publishing Tutorial
+
+ {% with github_config=settings("library.GitHubIntegrationConfiguration", use_default_site=True) %}
+ {% if github_config.can_use_github_integration(user) %}
+
+ GitHub
+
+ {% endif %}
+ {% endwith %}
+
+ Peer Review & DOIs
+
+
+
+
+
+ Our mission is to help computational modelers develop, document, and share their computational models in
+ accordance with community standards and
+ good open science and software engineering
+ practices. Model authors can publish their model source code in the Computational Model Library with narrative documentation as well as metadata
+ that supports open science and emerging norms that facilitate software citation, computational
+ reproducibility / frictionless reuse, and interoperability. Model authors can also request private peer review of their computational models. Models that pass peer review
+ receive a DOI once published.
+
+
+ All users of models published in the library must cite model authors when they use and benefit
+ from their code.
+
+
+ Please check out our model publishing
+ tutorial and feel free to contact us if you have any
+ questions or concerns about publishing your model(s) in the Computational Model Library.
+
+
-
- Please check out our model publishing tutorial and feel free to contact us if you have any questions or concerns about publishing your model(s) in the Computational Model Library.
-
-
- We also maintain a curated database of over 7500 publications of
- agent-based and individual based models with detailed metadata on availability of code and
- bibliometric information on the landscape of ABM/IBM publications that we welcome you to explore.
-
-
+
\ No newline at end of file
diff --git a/django/library/jinja2/library/review/dashboard.jinja b/django/library/jinja2/library/review/dashboard.jinja
index 9514800bd..77e6fcec7 100644
--- a/django/library/jinja2/library/review/dashboard.jinja
+++ b/django/library/jinja2/library/review/dashboard.jinja
@@ -1,6 +1,7 @@
{% extends 'sidebar_layout.jinja' %}
{% from "common.jinja" import breadcrumb, paginator %}
{% from "library/review/includes/macros.jinja" import display_closed_status %}
+{% from "library/codebases/macros.jinja" import imported_release_indicator %}
{% block introduction %}
Peer Review Editor Dashboard
@@ -32,6 +33,7 @@
{% set review = release.review %}
+ {{ imported_release_indicator(release) }}
Review for version {{ release.version_number }}
{{ display_closed_status(review) }}
diff --git a/django/library/jinja2/library/review/email/model_revisions_requested_imported.jinja b/django/library/jinja2/library/review/email/model_revisions_requested_imported.jinja
new file mode 100644
index 000000000..8364ca9aa
--- /dev/null
+++ b/django/library/jinja2/library/review/email/model_revisions_requested_imported.jinja
@@ -0,0 +1,23 @@
+{% set release=review.codebase_release %}
+{% set imported_sync_state = release.imported_release_sync_state %}
+
+Dear {{ review.contact_author_name }},
+
+The reviewer for your computational model [{{ release.title }}]({{ build_absolute_uri(release.share_url) }}) has recommended that changes be made to your model. These are summarized below:
+
+{% autoescape false %}
+{{ feedback.notes_to_author }}
+{% endautoescape %}
+
+Since this release is imported from GitHub, you will need to update it there and then re-import it into CoMSES Net. Here is the process:
+
+1. **Commit and push your changes:** Make the necessary changes to your code and push them to your GitHub repository.
+2. **Edit the release on GitHub:** Go to the [release page on GitHub]({{ imported_sync_state.html_url }}) to edit the release.
+3. **Update the release tag:**
+ - In the release edit form, you can choose which tag or branch the release should be associated with.
+ - To include your latest changes, you should create a new tag. You can do this by typing a new version number in the "Tag version" box and selecting "Create new tag: [new_version] on publish". Make sure this new tag points to the branch with your recent commits.
+ - Keep in mind that the original version number will be kept, so its recommended to append a suffix to the tag instead of incrementing, e.g. "v1.0.0-rev1"
+4. **Confirm the release has been updated:** Follow [this link]({{ build_absolute_uri(release.get_edit_url()) }}) to confirm that the release has been updated, making sure files are categorized and metadata is complete.
+5. **Notify reviewer:** Once changes have been made, use the "Notify Reviewer of Changes" button to send an alert to the reviewer.
+
+On behalf of the editors@comses.net, thank you for submitting your computational model(s) to CoMSES Net! Our peer review service is intended to serve the community and we hope that you find the requested changes will improve your model's accessibility and potential for reuse. If you have any questions or concerns about this process, please feel free to [contact us]({{ build_absolute_uri(slugurl("contact")) }}).
diff --git a/django/library/jinja2/library/review/feedback/update.jinja b/django/library/jinja2/library/review/feedback/update.jinja
index 46e746b6f..51ffdd17c 100644
--- a/django/library/jinja2/library/review/feedback/update.jinja
+++ b/django/library/jinja2/library/review/feedback/update.jinja
@@ -35,6 +35,12 @@
(click here to open in a new window/tab)
+ {% if release.is_imported %}
+
+
+ About the Model Library
-
-
-
+-
-
+ - Our mission is to help computational modelers develop, document, and share their computational models in accordance with community standards and good open science and software engineering practices. Model authors can publish their model source code in the Computational Model Library with narrative documentation as well as metadata that supports open science and emerging norms that facilitate software citation, computational reproducibility / frictionless reuse, and interoperability. Model authors can also request private peer review of their computational models. Models that pass peer review receive a DOI once published. -
-- All users of models published in the library must cite model authors when they use and benefit from their code. -
+ + Publishing Tutorial + + {% with github_config=settings("library.GitHubIntegrationConfiguration", use_default_site=True) %} + {% if github_config.can_use_github_integration(user) %} + + GitHub + + {% endif %} + {% endwith %} + + Peer Review & DOIs + +
+
-
+
+ Our mission is to help computational modelers develop, document, and share their computational models in + accordance with community standards and + good open science and software engineering + practices. Model authors can publish their model source code in the Computational Model Library with narrative documentation as well as metadata + that supports open science and emerging norms that facilitate software citation, computational + reproducibility / frictionless reuse, and interoperability. Model authors can also request private peer review of their computational models. Models that pass peer review + receive a DOI once published. +
++ All users of models published in the library must cite model authors when they use and benefit + from their code. +
++ Please check out our model publishing + tutorial and feel free to contact us if you have any + questions or concerns about publishing your model(s) in the Computational Model Library. +
+- Please check out our model publishing tutorial and feel free to contact us if you have any questions or concerns about publishing your model(s) in the Computational Model Library. -
-- We also maintain a curated database of over 7500 publications of - agent-based and individual based models with detailed metadata on availability of code and - bibliometric information on the landscape of ABM/IBM publications that we welcome you to explore. -
-+ Please note that this model release was imported from GitHub, so the file structure may differ from + a typical release package in the CoMSES Model Library. +
+ {% endif %}
You can save your progress at any time with the save button. Use the Submit button to send your finalized
diff --git a/django/library/jinja2/library/review/invitations/update.jinja b/django/library/jinja2/library/review/invitations/update.jinja
index 3e5174f17..f2aa5de99 100644
--- a/django/library/jinja2/library/review/invitations/update.jinja
+++ b/django/library/jinja2/library/review/invitations/update.jinja
@@ -29,6 +29,12 @@
If this model is not published, please do not share it with anyone else.
+ {% if release.is_imported %}
+
diff --git a/django/library/management/commands/update_codebase_metadata.py b/django/library/management/commands/update_codebase_metadata.py
index 9f3f1a40d..d86bc4987 100644
--- a/django/library/management/commands/update_codebase_metadata.py
+++ b/django/library/management/commands/update_codebase_metadata.py
@@ -53,7 +53,7 @@ def update_release_packages(self):
for release in releases:
try:
fs_api = release.get_fs_api()
- fs_api.rebuild(metadata_only=True)
+ fs_api.rebuild_metadata()
except Exception as e:
errors.append((release, e))
if errors:
diff --git a/django/library/metadata.py b/django/library/metadata.py
index 535f5f46d..5a482cbcd 100644
--- a/django/library/metadata.py
+++ b/django/library/metadata.py
@@ -140,7 +140,7 @@ def _common_codebase_fields(cls, codebase) -> dict:
return dict(
type_="SoftwareSourceCode",
name=codebase.title,
- codeRepository=codebase.repository_url or None,
+ codeRepository=codebase.code_repository or None,
applicationCategory="Computational Model",
citation=[
cls.to_textual_creative_work(text)
@@ -388,19 +388,205 @@ def convert_codebase(cls, codebase, codemeta: CodeMeta | dict = None) -> DataCit
)
-def coerce_codemeta(codemeta: dict | CodeMeta, codebase=None, release=None) -> CodeMeta:
- """make sure that codemeta is a CodeMeta object. If we didn't receive anything,
- try to re-generate it from whichever object is given (codebase or release)"""
+class ReleaseMetadataConverter:
+ """Extract CoMSES CodebaseRelease metadata from various sources."""
+
+ def __init__(
+ self,
+ codemeta: CodeMeta | dict | None = None,
+ cff: CitationFileFormat | dict | None = None,
+ github_repository: dict | None = None,
+ github_release: dict | None = None,
+ ) -> dict:
+ self.codemeta = coerce_codemeta(codemeta)
+ self.cff = coerce_cff(cff)
+ self.github_repository = github_repository
+ self.github_release = github_release
+
+ def _get_field(self, source, key: str):
+ """helper to retrieve a field from either a dict or object, returning
+ None if the source either doesn't exist or doesn't contain the field"""
+ if source is None:
+ return None
+ if isinstance(source, dict):
+ return source.get(key, None)
+ return getattr(source, key, None)
+
+ def _ensure_list(self, value):
+ """return a list containing the value if it is not a list,
+ or the value if it is already a list"""
+ if isinstance(value, list):
+ return value
+ elif value is None:
+ return []
+ return [value]
+
+ def _extract_license_from_github(self) -> str | None:
+ gh_license = self._get_field(self.github_repository, "license")
+ if isinstance(gh_license, dict):
+ spdx_id = gh_license.get("spdx_id", None)
+ if spdx_id:
+ return spdx_id
+ return None
+
+ def _extract_license_from_cff(self) -> str | None:
+ cff_licenses = self._ensure_list(self._get_field(self.cff, "license"))
+ for cff_license in cff_licenses:
+ if cff_license and hasattr(cff_license, "value"):
+ return cff_license.value
+ return None
+
+ def _extract_release_notes_from_codemeta(self) -> str | None:
+ release_notes = self._get_field(self.codemeta, "releaseNotes")
+ if release_notes:
+ if isinstance(release_notes, list):
+ release_notes = release_notes[0]
+ return release_notes if isinstance(release_notes, str) else None
+ elif isinstance(release_notes, str):
+ return release_notes
+ return None
+
+ def _extract_release_notes_from_github(self) -> str | None:
+ body = self._get_field(self.github_release, "body")
+ return body if isinstance(body, str) else None
+
+ def _extract_os_from_codemeta(self) -> str:
+ codemeta_os = self._get_field(self.codemeta, "operatingSystem")
+ if not codemeta_os:
+ return ""
+ if isinstance(codemeta_os, list):
+ if not codemeta_os:
+ return ""
+ os_value = codemeta_os[0]
+ else:
+ os_value = codemeta_os
+
+ if isinstance(os_value, str):
+ os_str = os_value
+ else:
+ # try to extract a name from a structured object (e.g., {"name": "Linux"})
+ os_name = self._get_field(os_value, "name")
+ os_str = os_name if isinstance(os_name, str) else ""
+
+ if os_str:
+ normalized = os_str.lower()
+ # attempt to match the given os string to a known platform
+ if "linux" in normalized:
+ return "linux"
+ elif "mac" in normalized or "os x" in normalized:
+ return "macos"
+ elif "windows" in normalized:
+ return "windows"
+ elif "independent" in normalized or "any" in normalized:
+ return "platform_independent"
+ else:
+ return "other"
+ return ""
+
+ def _extract_programming_languages_from_codemeta(self) -> list[str] | None:
+ codemeta_langs = self._ensure_list(
+ self._get_field(self.codemeta, "programmingLanguage")
+ )
+ langs = []
+ for codemeta_lang in codemeta_langs:
+ if isinstance(codemeta_lang, str):
+ langs.append(codemeta_lang)
+ else: # try to get the name from a ComputerLanguage object
+ lang_name = self._get_field(codemeta_lang, "name")
+ if lang_name:
+ langs.append(lang_name)
+ return langs if langs else None
+
+ def _extract_programming_languages_from_github(self) -> list[str] | None:
+ gh_lang = self._get_field(self.github_repository, "language")
+ if gh_lang and isinstance(gh_lang, str):
+ return [gh_lang]
+ return None
+
+ def _extract_platforms_from_codemeta(self) -> list[str] | None:
+ runtime_platforms = self._ensure_list(
+ self._get_field(self.codemeta, "runtimePlatform")
+ )
+ platforms = []
+ for runtime_platform in runtime_platforms:
+ if isinstance(runtime_platform, str):
+ platforms.append(runtime_platform)
+ return platforms if platforms else None
+
+ def convert(self) -> dict:
+ """return a dictionary with the metadata fields for a codebase release from
+ given sources"""
+ # set priority
+ license_spdx_id = (
+ self._extract_license_from_github() or self._extract_license_from_cff()
+ )
+ release_notes = (
+ self._extract_release_notes_from_codemeta()
+ or self._extract_release_notes_from_github()
+ )
+ programming_languages = self._extract_programming_languages_from_github()
+
+ return {
+ "license_spdx_id": license_spdx_id,
+ "release_notes": release_notes,
+ "os": self._extract_os_from_codemeta(),
+ "programming_languages": programming_languages,
+ "platforms": self._extract_platforms_from_codemeta(),
+ }
+
+
+def coerce_codemeta(
+ codemeta: dict | CodeMeta | None, codebase=None, release=None
+) -> CodeMeta | None:
+ """
+ Ensure the returned value is a CodeMeta object or None.
+ - if given a CodeMeta instance, return it
+ - if given a dictionary, parse it into a CodeMeta object
+ - if given None, attempt to generate codemeta from the given codebase or release
+ - return None if the input cannot be converted or generated
+ """
if isinstance(codemeta, CodeMeta):
return codemeta
- if codebase is None:
+ if isinstance(codemeta, dict):
+ try:
+ return CodeMeta(**codemeta)
+ except Exception:
+ logger.exception("Error when parsing codemeta from dict: %s", codemeta)
+ return None
+ if codemeta is None:
if codebase:
return CodeMetaConverter.convert_codebase(codebase)
elif release:
return CodeMetaConverter.convert_release(release)
- if isinstance(codemeta, dict):
+ logger.debug(
+ "unable to coerce codemeta from inputs: codemeta=%s, codebase=%s, release=%s",
+ codemeta,
+ codebase,
+ release,
+ )
+ return None
+
+
+def coerce_cff(
+ cff: dict | CitationFileFormat | None, release=None
+) -> CitationFileFormat | None:
+ """
+ Ensure the returned value is a CitationFileFormat object or None.
+ - if given a CitationFileFormat instance, return it
+ - if given a dictionary, parse it into a CitationFileFormat object
+ - if given None, attempt to generate cff from the given release
+ - return None if the input cannot be converted or generated
+ """
+ if isinstance(cff, CitationFileFormat):
+ return cff
+ if isinstance(cff, dict):
try:
- return CodeMeta(**codemeta)
- except Exception as e:
- raise ValueError("Invalid codemeta dictionary") from e
- raise TypeError("codemeta must be a valid dictionary or CodeMeta instance")
+ return CitationFileFormat(**cff)
+ except Exception:
+ logger.exception("Error when parsing cff from dict: %s", cff)
+ return None
+ if cff is None:
+ if release:
+ return CitationFileFormatConverter.convert_release(release)
+ logger.debug("unable to coerce cff from inputs: cff=%s, release=%s", cff, release)
+ return None
diff --git a/django/library/migrations/0036_codebasegitremote_githubintegrationappinstallation_and_more.py b/django/library/migrations/0036_codebasegitremote_githubintegrationappinstallation_and_more.py
new file mode 100644
index 000000000..412959478
--- /dev/null
+++ b/django/library/migrations/0036_codebasegitremote_githubintegrationappinstallation_and_more.py
@@ -0,0 +1,670 @@
+# Generated by Django 5.2.11 on 2026-03-09 21:05
+
+import core.fields
+import django.db.models.deletion
+import modelcluster.fields
+import modelsearch.index
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("library", "0035_alter_releaselanguage_options_and_more"),
+ ("wagtailcore", "0096_referenceindex_referenceindex_source_object_and_more"),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="CodebaseGitRemote",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "is_user_repo",
+ models.BooleanField(
+ default=False, help_text="Whether this is a user-owned remote"
+ ),
+ ),
+ (
+ "is_preexisting",
+ models.BooleanField(
+ default=False,
+ help_text="Whether this remote was pre-existing or based on a codebase git mirror",
+ ),
+ ),
+ ("repo_name", models.CharField(max_length=100)),
+ (
+ "owner",
+ models.CharField(
+ help_text="Github username (or organization) who owns the remote repository",
+ max_length=100,
+ ),
+ ),
+ (
+ "url",
+ models.URLField(
+ blank=True, help_text="http URL of remote repository"
+ ),
+ ),
+ ("is_active", models.BooleanField(default=False)),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "codebase",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="git_remotes",
+ to="library.codebase",
+ ),
+ ),
+ ],
+ ),
+ migrations.CreateModel(
+ name="GithubIntegrationAppInstallation",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("github_user_id", models.BigIntegerField(unique=True)),
+ ("github_login", models.CharField(max_length=255)),
+ ("installation_id", models.BigIntegerField(unique=True)),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "user",
+ models.OneToOneField(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="github_integration_app_installation",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ ],
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationConfiguration",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "is_beta",
+ models.BooleanField(
+ default=True,
+ help_text="Mark the GitHub integration feature as a beta feature",
+ ),
+ ),
+ (
+ "is_enabled_globally",
+ models.BooleanField(
+ default=False,
+ help_text="Enable GitHub integration for all users. When enabled, approved_users whitelist is ignored",
+ ),
+ ),
+ (
+ "approved_users",
+ models.ManyToManyField(
+ blank=True,
+ help_text="Whitelist of users who can access GitHub integration. Superusers/staff are implicitly included",
+ related_name="github_integration_approved",
+ to=settings.AUTH_USER_MODEL,
+ ),
+ ),
+ (
+ "site",
+ models.OneToOneField(
+ editable=False,
+ on_delete=django.db.models.deletion.CASCADE,
+ to="wagtailcore.site",
+ ),
+ ),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationExistingRepoInfoItem",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "icon",
+ models.CharField(
+ blank=True,
+ help_text="optional icon class (e.g., 'fas fa-link')",
+ max_length=200,
+ ),
+ ),
+ (
+ "text",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="The info item text (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "text_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_text_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="existing_repo_info_items",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationExistingRepoTutorialVideo",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "title",
+ models.CharField(
+ blank=True,
+ help_text="optional short title for the tutorial",
+ max_length=200,
+ ),
+ ),
+ (
+ "youtube_id",
+ models.CharField(
+ help_text="YouTube video id (e.g., dQw4w9WgXcQ)", max_length=32
+ ),
+ ),
+ (
+ "caption",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="optional caption (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "caption_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_caption_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="existing_repo_tutorial_videos",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationFaqEntry",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "question",
+ models.CharField(help_text="The FAQ question", max_length=500),
+ ),
+ (
+ "answer",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="The FAQ answer in Markdown format",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "answer_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_answer_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="faq_entries",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationNewRepoInfoItem",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "icon",
+ models.CharField(
+ blank=True,
+ help_text="optional icon class (e.g., 'fas fa-link')",
+ max_length=200,
+ ),
+ ),
+ (
+ "text",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="The info item text (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "text_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_text_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="new_repo_info_items",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitHubIntegrationNewRepoTutorialVideo",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "sort_order",
+ models.IntegerField(blank=True, editable=False, null=True),
+ ),
+ (
+ "title",
+ models.CharField(
+ blank=True,
+ help_text="optional short title for the tutorial",
+ max_length=200,
+ ),
+ ),
+ (
+ "youtube_id",
+ models.CharField(
+ help_text="YouTube video id (e.g., dQw4w9WgXcQ)", max_length=32
+ ),
+ ),
+ (
+ "caption",
+ core.fields.MarkdownField(
+ blank=True,
+ help_text="optional caption (Markdown)",
+ rendered_field=True,
+ ),
+ ),
+ (
+ "caption_markup_type",
+ models.CharField(
+ choices=[
+ ("", "--"),
+ ("markdown", "markdown"),
+ ("html", "html"),
+ ("plain", "plain"),
+ ("", ""),
+ ],
+ default="markdown",
+ max_length=30,
+ ),
+ ),
+ ("_caption_rendered", models.TextField(editable=False)),
+ (
+ "configuration",
+ modelcluster.fields.ParentalKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="new_repo_tutorial_videos",
+ to="library.githubintegrationconfiguration",
+ ),
+ ),
+ ],
+ options={
+ "ordering": ["sort_order", "id"],
+ "abstract": False,
+ },
+ bases=(modelsearch.index.Indexed, models.Model),
+ ),
+ migrations.CreateModel(
+ name="GitRefSyncState",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "status",
+ models.CharField(
+ choices=[
+ ("PENDING", "pending"),
+ ("RUNNING", "running"),
+ ("SUCCESS", "success"),
+ ("ERROR", "error"),
+ ],
+ default="PENDING",
+ max_length=20,
+ ),
+ ),
+ ("last_log", models.TextField(blank=True)),
+ ("full_log", models.JSONField(default=list)),
+ ("error_message", models.TextField(blank=True)),
+ ("http_status", models.IntegerField(blank=True, null=True)),
+ ("last_sync_started_at", models.DateTimeField(blank=True, null=True)),
+ ("last_sync_finished_at", models.DateTimeField(blank=True, null=True)),
+ ("batch_id", models.CharField(blank=True, max_length=36, null=True)),
+ ("tag_name", models.CharField(blank=True, max_length=128)),
+ (
+ "built_commit_sha",
+ models.CharField(
+ blank=True,
+ help_text="Last commit SHA created for this git ref in the mirror repo",
+ max_length=40,
+ ),
+ ),
+ (
+ "last_built_at",
+ models.DateTimeField(
+ blank=True,
+ help_text="Updated when the git ref is committed to the mirror repo",
+ null=True,
+ ),
+ ),
+ (
+ "pushed_commit_sha",
+ models.CharField(
+ blank=True,
+ help_text="Last commit SHA pushed to the remote repo",
+ max_length=40,
+ ),
+ ),
+ ("branch_name", models.CharField(blank=True, max_length=128)),
+ (
+ "remote",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="git_ref_sync_states",
+ to="library.codebasegitremote",
+ ),
+ ),
+ ],
+ options={
+ "abstract": False,
+ },
+ ),
+ migrations.AddField(
+ model_name="codebase",
+ name="main_git_ref_sync_state",
+ field=models.OneToOneField(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="codebase",
+ to="library.gitrefsyncstate",
+ ),
+ ),
+ migrations.AddField(
+ model_name="codebaserelease",
+ name="git_ref_sync_state",
+ field=models.OneToOneField(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="release",
+ to="library.gitrefsyncstate",
+ ),
+ ),
+ migrations.CreateModel(
+ name="ImportedReleaseSyncState",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("date_created", models.DateTimeField(auto_now_add=True)),
+ ("last_modified", models.DateTimeField(auto_now=True)),
+ (
+ "status",
+ models.CharField(
+ choices=[
+ ("PENDING", "pending"),
+ ("RUNNING", "running"),
+ ("SUCCESS", "success"),
+ ("ERROR", "error"),
+ ],
+ default="PENDING",
+ max_length=20,
+ ),
+ ),
+ ("last_log", models.TextField(blank=True)),
+ ("full_log", models.JSONField(default=list)),
+ ("error_message", models.TextField(blank=True)),
+ ("http_status", models.IntegerField(blank=True, null=True)),
+ ("last_sync_started_at", models.DateTimeField(blank=True, null=True)),
+ ("last_sync_finished_at", models.DateTimeField(blank=True, null=True)),
+ ("batch_id", models.CharField(blank=True, max_length=36, null=True)),
+ ("tag_name", models.CharField(blank=True, max_length=128)),
+ (
+ "category_manifest",
+ models.JSONField(
+ default=dict,
+ help_text="Maps file paths to categories (code, docs, data, results)",
+ ),
+ ),
+ ("github_release_id", models.CharField(max_length=128)),
+ (
+ "display_name",
+ models.CharField(
+ blank=True,
+ help_text="A human-readable name for the release, if different from the tag name",
+ max_length=255,
+ ),
+ ),
+ (
+ "html_url",
+ models.URLField(
+ blank=True,
+ help_text="URL to the (browser-viewable) page for the release on the external service",
+ ),
+ ),
+ (
+ "download_url",
+ models.URLField(
+ blank=True,
+ help_text="URL to the downloadable package for the release on the external service, ideally a zipball such as in the case of github",
+ ),
+ ),
+ (
+ "extra_data",
+ models.JSONField(
+ default=dict,
+ help_text="Additional data for the external release, ideally the full object (e.g. github release) from the external service",
+ ),
+ ),
+ (
+ "remote",
+ models.ForeignKey(
+ help_text="Remote from which this release was imported",
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ related_name="imported_sync_states",
+ to="library.codebasegitremote",
+ ),
+ ),
+ ],
+ ),
+ migrations.AddField(
+ model_name="codebaserelease",
+ name="imported_release_sync_state",
+ field=models.OneToOneField(
+ null=True,
+ on_delete=django.db.models.deletion.SET_NULL,
+ to="library.importedreleasesyncstate",
+ ),
+ ),
+ migrations.AddConstraint(
+ model_name="codebasegitremote",
+ constraint=models.UniqueConstraint(
+ condition=models.Q(("is_active", True)),
+ fields=("codebase",),
+ name="single_active_remote",
+ ),
+ ),
+ migrations.AlterUniqueTogether(
+ name="codebasegitremote",
+ unique_together={("owner", "repo_name")},
+ ),
+ migrations.AlterUniqueTogether(
+ name="importedreleasesyncstate",
+ unique_together={("remote", "github_release_id")},
+ ),
+ ]
diff --git a/django/library/migrations/0037_initial_github_integration_overview_content.py b/django/library/migrations/0037_initial_github_integration_overview_content.py
new file mode 100644
index 000000000..f561de483
--- /dev/null
+++ b/django/library/migrations/0037_initial_github_integration_overview_content.py
@@ -0,0 +1,198 @@
+# Generated by Django 5.2.11 on 2026-03-09 21:06
+
+from django.db import migrations
+
+
+FAQ_ENTRIES = [
+ {
+ "question": "What GitHub permissions do you request and why?",
+ "answer": "We request read and write access to repository contents. You can either select specific repositories or allow access to all current and future repositories (though we will not attempt to access any repositories besides those connected).\r\n\r\nIf you choose the selection option, ensure that the repository you want to connect is included.",
+ },
+ {
+ "question": "Can I connect with a private GitHub repository?",
+ "answer": "No, only public GitHub repositories are supported at the moment.",
+ },
+ {
+ "question": "Can I connect with a repository owned by another user or organization?",
+ "answer": "Currently, no. We suggest forking the repository to your own account.",
+ },
+ {
+ "question": "If I have a repository connected to a model, what happens if I change the repository name, transfer ownership, make it private, or delete it?",
+ "answer": "If the repository is deleted or made private, you should disconnect the repository. If you need to change the name, you can disconnect and then reconnect with the new name. If the repository is transferred to a new owner, we suggest maintaining a fork of the repository.",
+ },
+]
+
+EXISTING_REPO_INFO_ITEMS = [
+ {
+ "icon": "fas fa-question",
+ "text": "Keep your workflow on GitHub and avoid duplicate uploading by importing tagged releases straight from GitHub",
+ },
+ {
+ "icon": "fas fa-file-archive",
+ "text": "GitHub release assets (the automatically generated zip file) are pulled into the model library and are available for download like any other release",
+ },
+ {
+ "icon": "fas fa-lock",
+ "text": "Releases imported from GitHub are private to begin with, letting you make sure everything looks right before requesting peer review or publishing",
+ },
+ {
+ "icon": "fas fa-file-alt",
+ "text": "Submissions are still required to contain source code and narrative documentation. We'll try to automatically categorize transferred files and let you make any corrections",
+ },
+ {
+ "icon": "fas fa-tags",
+ "text": "Software metadata like languages, license, etc. is extracted from the GitHub release and any existing `codemeta.json` or `CITATION.cff` files. Any missing fields can be filled in after importing",
+ },
+ {
+ "icon": "fas fa-clipboard-check",
+ "text": "Imported releases can be [peer reviewed](/reviews) just like any other submission, giving your model a DOI upon completion. Peer review **must** be requested **before** publishing.",
+ },
+ {
+ "icon": "fas fa-edit",
+ "text": "While unpublished or under review, you can still make changes to a release by updating the tag on GitHub and reimporting the release. A video tutorial demonstrates exactly how to do this.",
+ },
+]
+
+NEW_REPO_INFO_ITEMS = [
+ {
+ "icon": "fas fa-question",
+ "text": "Putting your model on GitHub makes it easier for authors and other researchers to keep track of changes and collaborate on development, while also boosting accessibility and unlocking an ecosystem of tools for making scientific software more reusable",
+ },
+ {
+ "icon": "fas fa-code-branch",
+ "text": "A git repository is automatically built for a model in the library by commiting **published** releases in order to a main branch, as well as seperate branches and tags for each release",
+ },
+ {
+ "icon": "fas fa-users",
+ "text": "In order to maintain attribution, the original submitter is set as the commit author, and all other authors as co-authors",
+ },
+ {
+ "icon": "fas fa-upload",
+ "text": "After connecting to a blank GitHub repository, you'll be able to push everything at once",
+ },
+ {
+ "icon": "fas fa-sync-alt",
+ "text": "New releases published in the model library or metadata changes to existing releases will be automatically added to the git repository but will require input to push to GitHub",
+ },
+ {
+ "icon": "fas fa-download",
+ "text": "We'll never overwrite any changes you make to any branch on the GitHub repository and if you make any new releases there, you'll be able to import them back into the model library",
+ },
+]
+
+EXISTING_REPO_TUTORIAL_VIDEOS = [
+ {
+ "title": "",
+ "youtube_id": "-",
+ "caption": "How to set up an existing repository and import releases into the model library",
+ },
+ {
+ "title": "",
+ "youtube_id": "-",
+ "caption": "How to update a GitHub release and re-import it",
+ },
+]
+
+NEW_REPO_TUTORIAL_VIDEOS = [
+ {
+ "title": "",
+ "youtube_id": "-",
+ "caption": "How to set up and push a new git repository for your model in the library",
+ }
+]
+
+
+def _get_github_config_for_default_site(apps):
+ Site = apps.get_model("wagtailcore", "Site")
+ GitHubIntegrationConfiguration = apps.get_model(
+ "library", "GitHubIntegrationConfiguration"
+ )
+ default_site = Site.objects.get(is_default_site=True)
+ config, _ = GitHubIntegrationConfiguration.objects.get_or_create(site=default_site)
+ return config
+
+
+def delete_existing_content(apps, schema_editor):
+ config = _get_github_config_for_default_site(apps)
+ apps.get_model("library", "GitHubIntegrationFaqEntry").objects.filter(
+ configuration=config
+ ).delete()
+ apps.get_model("library", "GitHubIntegrationExistingRepoInfoItem").objects.filter(
+ configuration=config
+ ).delete()
+ apps.get_model("library", "GitHubIntegrationNewRepoInfoItem").objects.filter(
+ configuration=config
+ ).delete()
+ apps.get_model(
+ "library", "GitHubIntegrationExistingRepoTutorialVideo"
+ ).objects.filter(configuration=config).delete()
+ apps.get_model("library", "GitHubIntegrationNewRepoTutorialVideo").objects.filter(
+ configuration=config
+ ).delete()
+
+
+def populate_initial_content(apps, schema_editor):
+ config = _get_github_config_for_default_site(apps)
+
+ GitHubIntegrationFaqEntry = apps.get_model("library", "GitHubIntegrationFaqEntry")
+ GitHubIntegrationExistingRepoInfoItem = apps.get_model(
+ "library", "GitHubIntegrationExistingRepoInfoItem"
+ )
+ GitHubIntegrationNewRepoInfoItem = apps.get_model(
+ "library", "GitHubIntegrationNewRepoInfoItem"
+ )
+ GitHubIntegrationExistingRepoTutorialVideo = apps.get_model(
+ "library", "GitHubIntegrationExistingRepoTutorialVideo"
+ )
+ GitHubIntegrationNewRepoTutorialVideo = apps.get_model(
+ "library", "GitHubIntegrationNewRepoTutorialVideo"
+ )
+
+ for sort_order, row in enumerate(FAQ_ENTRIES):
+ GitHubIntegrationFaqEntry.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ question=row["question"],
+ answer=row["answer"],
+ )
+ for sort_order, row in enumerate(EXISTING_REPO_INFO_ITEMS):
+ GitHubIntegrationExistingRepoInfoItem.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ icon=row["icon"],
+ text=row["text"],
+ )
+ for sort_order, row in enumerate(NEW_REPO_INFO_ITEMS):
+ GitHubIntegrationNewRepoInfoItem.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ icon=row["icon"],
+ text=row["text"],
+ )
+ for sort_order, row in enumerate(EXISTING_REPO_TUTORIAL_VIDEOS):
+ GitHubIntegrationExistingRepoTutorialVideo.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ title=row["title"],
+ youtube_id=row["youtube_id"],
+ caption=row["caption"],
+ )
+ for sort_order, row in enumerate(NEW_REPO_TUTORIAL_VIDEOS):
+ GitHubIntegrationNewRepoTutorialVideo.objects.create(
+ configuration=config,
+ sort_order=sort_order,
+ title=row["title"],
+ youtube_id=row["youtube_id"],
+ caption=row["caption"],
+ )
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("library", "0036_codebasegitremote_githubintegrationappinstallation_and_more"),
+ ]
+
+ operations = [
+ migrations.RunPython(populate_initial_content, delete_existing_content),
+ ]
diff --git a/django/library/models.py b/django/library/models.py
index e91846749..500936412 100644
--- a/django/library/models.py
+++ b/django/library/models.py
@@ -1,4 +1,5 @@
import hashlib
+import io
import json
import yaml
import logging
@@ -13,11 +14,13 @@
from collections import defaultdict
from datetime import date, timedelta
+from allauth.socialaccount.models import SocialAccount
from django.conf import settings
+from django.contrib.admin.widgets import FilteredSelectMultiple
from django.contrib.postgres.fields import ArrayField
from django.core.files.images import ImageFile
from django.core.files.storage import FileSystemStorage
-from django.db import models, transaction
+from django.db import IntegrityError, models, transaction
from django.db.models import Prefetch, Q, Count, Max
from django.utils.functional import cached_property
from django.urls import reverse
@@ -30,7 +33,7 @@
from modelcluster.models import ClusterableModel
from rest_framework.exceptions import ValidationError, UnsupportedMediaType
from taggit.models import TaggedItemBase
-from wagtail.admin.panels import FieldPanel
+from wagtail.admin.panels import FieldPanel, InlinePanel, HelpPanel
from wagtail.coreutils import string_to_ascii
from wagtail.images.models import (
Image,
@@ -40,8 +43,11 @@
get_upload_to,
ImageQuerySet,
)
+from wagtail.models import Orderable
from wagtail.search import index
from wagtail.snippets.models import register_snippet
+from wagtail.contrib.settings.models import BaseSiteSetting, register_setting
+from django.utils.html import format_html
from core import fs
from core.backends import add_to_comses_permission_whitelist
@@ -52,8 +58,9 @@
from core.view_helpers import get_search_queryset
from .fs import (
CodebaseReleaseFsApi,
+ ImportedCodebaseReleaseFsApi,
StagingDirectories,
- FileCategoryDirectories,
+ FileCategories,
MessageLevels,
)
from .metadata import CodeMetaConverter, DataCiteConverter, CitationFileFormatConverter
@@ -616,6 +623,427 @@ def updated_after(self, start_date, end_date=None, **kwargs):
return new_codebases, updated_codebases, releases
+class GithubIntegrationAppInstallation(models.Model):
+ """Represents a user's installation of the Github Integration App"""
+
+ user = models.OneToOneField(
+ settings.AUTH_USER_MODEL,
+ related_name="github_integration_app_installation",
+ on_delete=models.CASCADE,
+ )
+ github_user_id = models.BigIntegerField(unique=True)
+ github_login = models.CharField(max_length=255)
+ installation_id = models.BigIntegerField(unique=True)
+ date_created = models.DateTimeField(auto_now_add=True)
+ last_modified = models.DateTimeField(auto_now=True)
+
+ def __str__(self):
+ return f"{self.github_login} (installation_id={self.installation_id})"
+
+
+class CodebaseGitRemote(models.Model):
+ """Represents a remote repository associated with a `Codebase` that can be pushed to or imported from"""
+
+ is_user_repo = models.BooleanField(
+ default=False, help_text="Whether this is a user-owned remote"
+ )
+ is_preexisting = models.BooleanField(
+ default=False,
+ help_text="Whether this remote was pre-existing or based on a codebase git mirror",
+ )
+ repo_name = models.CharField(max_length=100)
+ owner = models.CharField(
+ max_length=100,
+ help_text="Github username (or organization) who owns the remote repository",
+ )
+ url = models.URLField(
+ blank=True,
+ help_text=_("http URL of remote repository"),
+ )
+ is_active = models.BooleanField(default=False)
+ codebase = models.ForeignKey(
+ "Codebase",
+ related_name="git_remotes",
+ on_delete=models.CASCADE,
+ )
+ date_created = models.DateTimeField(auto_now_add=True)
+ last_modified = models.DateTimeField(auto_now=True)
+
+ @property
+ def installation(self) -> GithubIntegrationAppInstallation | None:
+ if self.is_user_repo:
+ return self.codebase.submitter.github_integration_app_installation
+ return None
+
+ def __str__(self):
+ return f"{self.owner}/{self.repo_name} (active={self.is_active}))"
+
+ class Meta:
+ unique_together = ("owner", "repo_name")
+ constraints = [
+ models.UniqueConstraint(
+ fields=["codebase"],
+ condition=models.Q(is_active=True),
+ name="single_active_remote",
+ ),
+ ]
+
+
+class BaseReleaseSyncState(models.Model):
+ class Status(models.TextChoices):
+ PENDING = "PENDING", _("pending")
+ RUNNING = "RUNNING", _("running")
+ SUCCESS = "SUCCESS", _("success")
+ ERROR = "ERROR", _("error")
+
+ date_created = models.DateTimeField(auto_now_add=True)
+ last_modified = models.DateTimeField(auto_now=True)
+ status = models.CharField(
+ max_length=20, choices=Status.choices, default=Status.PENDING
+ )
+ last_log = models.TextField(blank=True)
+ full_log = models.JSONField(default=list)
+ error_message = models.TextField(blank=True)
+ http_status = models.IntegerField(null=True, blank=True)
+ last_sync_started_at = models.DateTimeField(null=True, blank=True)
+ last_sync_finished_at = models.DateTimeField(null=True, blank=True)
+ batch_id = models.CharField(max_length=36, blank=True, null=True)
+ tag_name = models.CharField(max_length=128, blank=True)
+
+ def __str__(self):
+ return f"status: {self.status}, last_log: {self.last_log}, error_message: {self.error_message}"
+
+ def _append_log(self, message: str, now):
+ logs = list(self.full_log or [])
+ logs.append({"t": now.isoformat(), "msg": message})
+ self.full_log = logs
+ self.last_log = message
+
+ def mark_running(self):
+ """set status to RUNNING and stamp last_sync_started_at"""
+ self.status = self.Status.RUNNING
+ self.last_sync_started_at = timezone.now()
+ self.save(update_fields=["status", "last_sync_started_at", "last_modified"])
+ return self
+
+ def log_failure(self, message: str):
+ """mark this sync state as ERROR and append a log entry"""
+ now = timezone.now()
+ self.status = self.Status.ERROR
+ self.error_message = message
+ self.last_sync_finished_at = now
+ self._append_log(message, now)
+ self.save()
+ return self
+
+ def log_success(self, message: str | None = None):
+ """mark this sync state as SUCCESS and optionally append a log entry"""
+ now = timezone.now()
+ self.status = self.Status.SUCCESS
+ self.last_sync_finished_at = now
+ if message:
+ self._append_log(message, now)
+ self.save()
+ return self
+
+ class Meta:
+ abstract = True
+
+
+class GitRefSyncState(BaseReleaseSyncState):
+ """
+ Represents the sync state (to a remote) of a git ref (release branch/tag or the codebase main branch)
+ that was built from a submission
+
+ exactly one is attached to either a CodebaseRelease (release branch / tag)
+ or a Codebase (main branch)
+
+ created with remote=None and assigned to a remote once a push is attempted, when swapping remotes,
+ the state is reset and assigned to the new remote
+ """
+
+ remote = models.ForeignKey(
+ "library.CodebaseGitRemote",
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ related_name="git_ref_sync_states",
+ )
+ built_commit_sha = models.CharField(
+ max_length=40,
+ blank=True,
+ help_text=_("Last commit SHA created for this git ref in the mirror repo"),
+ )
+ last_built_at = models.DateTimeField(
+ null=True,
+ blank=True,
+ help_text=_("Updated when the git ref is committed to the mirror repo"),
+ )
+ pushed_commit_sha = models.CharField(
+ max_length=40,
+ blank=True,
+ help_text=_("Last commit SHA pushed to the remote repo"),
+ )
+ branch_name = models.CharField(max_length=128, blank=True)
+
+ @classmethod
+ def latest_release_ref_for_codebase(cls, codebase):
+ return (
+ cls.objects.filter(release__codebase=codebase)
+ .select_related("release")
+ .order_by("release__version_number")
+ .last()
+ )
+
+ @classmethod
+ def releases_without_git_ref_sync_state(cls, codebase):
+ built_ids = cls.objects.filter(release__codebase=codebase).values_list(
+ "release__id", flat=True
+ )
+ return codebase.releases.internal().public().exclude(id__in=built_ids)
+
+ @classmethod
+ def for_release(cls, release):
+ """Fetch or create the release's git ref sync state."""
+ state = getattr(release, "git_ref_sync_state", None)
+ if state:
+ return state
+ state = cls.objects.create(status=cls.Status.PENDING)
+ release.git_ref_sync_state = state
+ release.save(
+ rebuild_metadata=False,
+ update_fields=["git_ref_sync_state", "last_modified"],
+ )
+ return state
+
+ @classmethod
+ def for_codebase(cls, codebase, branch_name="main"):
+ """Fetch or create the codebase main branch git ref sync state."""
+ state = getattr(codebase, "main_git_ref_sync_state", None)
+ if state:
+ return state
+ state = cls.objects.create(status=cls.Status.PENDING, branch_name=branch_name)
+ codebase.main_git_ref_sync_state = state
+ codebase.save(
+ rebuild_metadata=False,
+ rebuild_release_metadata=False,
+ update_fields=["main_git_ref_sync_state", "last_modified"],
+ )
+ return state
+
+ @classmethod
+ def get_last_remote_for_codebase(cls, codebase):
+ """return the remote that git ref sync states are currently bound to, if any.
+
+ checks the main branch state first, then falls back to the first release state.
+ returns None if no states have a remote assigned.
+ """
+ main_state = getattr(codebase, "main_git_ref_sync_state", None)
+ if main_state and main_state.remote:
+ return main_state.remote
+ release_state = cls.objects.filter(
+ release__codebase=codebase, remote__isnull=False
+ ).first()
+ if release_state:
+ return release_state.remote
+ return None
+
+ @classmethod
+ def reassign_remotes_for_codebase(cls, codebase, remote):
+ """reassign all git ref sync states for a codebase to a new remote and
+ DESTROY push state for the prior remote"""
+ qs = cls.objects.filter(release__codebase=codebase)
+ for state in qs:
+ state.reassign_remote(remote)
+ main_state = getattr(codebase, "main_git_ref_sync_state", None)
+ if main_state:
+ main_state.reassign_remote(remote)
+
+ @classmethod
+ def start_all_push_jobs_for_codebase(cls, codebase, remote) -> bool:
+ """set remotes and mark pushable refs (main + releases) as RUNNING"""
+ started = False
+ # start main branch job if needed
+ main_state = codebase.get_or_create_main_git_ref_sync_state()
+ main_state.set_remote(remote)
+ if main_state.can_push():
+ main_state.mark_running()
+ started = True
+ # start release jobs that can be pushed
+ releases = codebase.ordered_releases_list(internal_only=True, asc=True)
+ for release in releases:
+ state = release.get_or_create_git_ref_sync_state()
+ state.set_remote(remote)
+ if state.can_push():
+ state.mark_running()
+ started = True
+ return started
+
+ def reassign_remote(self, remote):
+ """bind to a new remote and reset push status/logs"""
+ self.remote = remote
+ self.pushed_commit_sha = ""
+ self.status = self.Status.PENDING
+ self.last_log = ""
+ self.error_message = ""
+ self.http_status = None
+ self.last_sync_started_at = None
+ self.last_sync_finished_at = None
+ # keeping this for now, might be useful if something goes wrong
+ # self.full_log = []
+ self.save()
+ return self
+
+ def record_build(self, commit_sha, tag_name=None, branch_name=None):
+ """record the latest commit hash and optionally tag/branch names if given and not already set"""
+ self.built_commit_sha = commit_sha
+ # only update if given and not already set
+ if tag_name and not (self.tag_name or "").strip():
+ self.tag_name = tag_name
+ if branch_name and not (self.branch_name or "").strip():
+ self.branch_name = branch_name
+ self.last_built_at = timezone.now()
+ self.save()
+ return self
+
+ def set_remote(self, remote):
+ """set the remote, only if not already set to something different"""
+ if self.remote and self.remote.id != remote.id:
+ raise ValueError("git ref sync state already bound to a different remote")
+ self.remote = remote
+ self.save()
+ return self
+
+ def record_push_success(self, commit_sha, summary=None):
+ self.pushed_commit_sha = commit_sha
+ self.save(update_fields=["pushed_commit_sha", "last_modified"])
+ return self.log_success(summary)
+
+ def can_push(self) -> bool:
+ """
+ return true when a git ref is pushable, either:
+ - push failed
+ - no commit sha has been pushed
+ - or the pushed sha is out of date
+ """
+ if self.status == self.Status.ERROR:
+ return True
+ if not self.pushed_commit_sha:
+ return True
+ if self.built_commit_sha and self.built_commit_sha != self.pushed_commit_sha:
+ return True
+ return False
+
+
+class ImportedReleaseSyncState(BaseReleaseSyncState):
+ """Represents the state of the process involved in importing a release from GitHub.
+ Will only exist for releases that have at least been attempted to be imported from GitHub
+
+ only one will exist for a given CodebaseRelease, since a release can only be imported once.
+ though multiple can exist for a GitHub release, only if they are from different remotes.
+ """
+
+ # remote is the source repository this GitHub release came from
+ remote = models.ForeignKey(
+ "library.CodebaseGitRemote",
+ null=True,
+ on_delete=models.SET_NULL,
+ related_name="imported_sync_states",
+ help_text=_("Remote from which this release was imported"),
+ )
+ category_manifest = models.JSONField(
+ default=dict,
+ help_text="Maps file paths to categories (code, docs, data, results)",
+ )
+ github_release_id = models.CharField(
+ max_length=128,
+ )
+ display_name = models.CharField(
+ max_length=255,
+ blank=True,
+ help_text="A human-readable name for the release, if different from the tag name",
+ )
+ html_url = models.URLField(
+ blank=True,
+ help_text="URL to the (browser-viewable) page for the release on the external service",
+ )
+ download_url = models.URLField(
+ blank=True,
+ help_text="URL to the downloadable package for the release on the external service, ideally a zipball such as in the case of github",
+ )
+ extra_data = models.JSONField(
+ default=dict,
+ help_text="Additional data for the external release, ideally the full object (e.g. github release) from the external service",
+ )
+
+ class Meta:
+ unique_together = ("remote", "github_release_id")
+
+ # manager-style helpers
+ @classmethod
+ def for_github_release(cls, remote, gh_release_raw: dict):
+ """create or update a sync state from a github release object from the github API.
+ only updates when tracked fields change
+ """
+ release_id = str(gh_release_raw.get("id", ""))
+ if not release_id:
+ raise ValueError("missing github release id")
+ tag_name = gh_release_raw.get("tag_name")
+ display_name = gh_release_raw.get("name")
+ html_url = gh_release_raw.get("html_url")
+ download_url = gh_release_raw.get("zipball_url")
+
+ obj, _ = cls.objects.get_or_create(
+ remote=remote,
+ github_release_id=release_id,
+ defaults={
+ "tag_name": tag_name,
+ "display_name": display_name,
+ "html_url": html_url,
+ "download_url": download_url,
+ "extra_data": gh_release_raw,
+ "status": cls.Status.PENDING,
+ },
+ )
+ changed = False
+ if obj.tag_name != tag_name:
+ obj.tag_name = tag_name
+ changed = True
+ if obj.display_name != display_name:
+ obj.display_name = display_name
+ changed = True
+ if obj.html_url != html_url:
+ obj.html_url = html_url
+ changed = True
+ if obj.download_url != download_url:
+ obj.download_url = download_url
+ changed = True
+ # store full release object
+ if (obj.extra_data or {}) != (gh_release_raw or {}):
+ obj.extra_data = gh_release_raw or {}
+ changed = True
+
+ if changed:
+ obj.save()
+ return obj
+
+ def can_reimport(self) -> bool:
+ """
+ return true when a re-import should be allowed:
+ - job has completed (SUCCESS) or failed (ERROR)
+ - and either no CodebaseRelease exists yet for this sync state
+ - or the existing CodebaseRelease is not live (i.e., not published)
+ """
+ if self.status not in [self.Status.SUCCESS, self.Status.ERROR]:
+ return False
+ release = CodebaseRelease.objects.filter(
+ imported_release_sync_state=self
+ ).first()
+ if not release:
+ return True
+ return not release.live
+
+
@add_to_comses_permission_whitelist
class Codebase(index.Indexed, ModeratedContent, ClusterableModel):
"""
@@ -699,6 +1127,13 @@ class Codebase(index.Indexed, ModeratedContent, ClusterableModel):
"JSON metadata conforming to the codemeta schema. Cached as of the last update"
),
)
+ main_git_ref_sync_state = models.OneToOneField(
+ "library.GitRefSyncState",
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ related_name="codebase",
+ )
objects = CodebaseQuerySet.as_manager()
@@ -747,6 +1182,13 @@ def datacite_temp(self):
self, codemeta=self.codemeta_snapshot or None
)
+ @property
+ def code_repository(self):
+ git_remote = self.active_git_remote
+ if self.pk and git_remote:
+ return git_remote.url
+ return self.repository_url
+
@property
def is_replication(self):
return bool(self.replication_text.strip())
@@ -812,6 +1254,10 @@ def media_dir(self, *args):
def relative_media_path(self, *args):
return pathlib.Path(str(self.uuid), "media", *args)
+ @property
+ def active_git_remote(self):
+ return self.git_remotes.filter(is_active=True).first()
+
@property
def summarized_description(self):
if self.summary:
@@ -833,6 +1279,38 @@ def base_library_dir(self):
def base_git_dir(self):
return pathlib.Path(settings.REPOSITORY_ROOT, str(self.uuid))
+ def get_or_create_main_git_ref_sync_state(self, branch_name="main"):
+ """return the git ref sync state for this codebase's main branch"""
+ return GitRefSyncState.for_codebase(self, branch_name)
+
+ def create_remote(self, owner, repo_name, **kwargs):
+ """create a new remote for this codebase, or get an existing one if there is a
+ matching remote that was previously deactivated"""
+ existing_remote = CodebaseGitRemote.objects.filter(
+ codebase=self,
+ owner=owner,
+ repo_name=repo_name,
+ ).first()
+ if existing_remote:
+ existing_remote.is_active = True
+ existing_remote.save(update_fields=["is_active", "last_modified"])
+ return existing_remote
+ remote = CodebaseGitRemote.objects.create(
+ codebase=self,
+ owner=owner,
+ repo_name=repo_name,
+ **kwargs,
+ )
+ return remote
+
+ def latest_release_git_ref_sync_state(self):
+ """return the latest (by version number) git ref sync state for this codebase"""
+ return GitRefSyncState.latest_release_ref_for_codebase(self)
+
+ def releases_without_git_ref_sync_state(self):
+ """return internal (non-github imported), public releases that have not yet been added to the git repo"""
+ return GitRefSyncState.releases_without_git_ref_sync_state(self)
+
@property
def publication_year(self):
return (
@@ -873,6 +1351,8 @@ def _get_unique_contributors(self, release_contributors):
"""
Returns a unique, unordered set of Contributor objects given an initial filtered set of ReleaseContributors for this Codebase.
"""
+ if not self.pk:
+ return Contributor.objects.none()
return Contributor.objects.filter(
id__in=release_contributors.for_codebase(self).values("contributor_id")
)
@@ -903,21 +1383,23 @@ def download_count(self):
release__codebase__id=self.id
).count()
- def ordered_releases_list(self, has_change_perm=False, asc=True, **kwargs):
+ def ordered_releases_list(
+ self, has_change_perm=False, asc=True, internal_only=False, **kwargs
+ ):
"""
- list public releases (or all release if has_change_perm is True) in ascending (default) or descending order
+ list public releases (or all release if has_change_perm is True) in ascending (default) or descending order.
+ If internal_only is True, only return internal (directly uploaded) releases
"""
if has_change_perm:
releases = self.releases.filter(**kwargs)
else:
- releases = self.public_releases(**kwargs)
+ releases = self.releases.public(**kwargs)
+ if internal_only:
+ releases = releases.internal()
releases_list = list(releases)
releases_list.sort(key=lambda r: Version(r.version_number), reverse=not asc)
return releases_list
- def public_releases(self, **kwargs):
- return self.releases.filter(status=CodebaseRelease.Status.PUBLISHED, **kwargs)
-
@classmethod
def get_list_url(cls):
return reverse("library:codebase-list")
@@ -1030,14 +1512,20 @@ def import_media(self, fileobj, user=None, title=None, images_only=True):
return image_metadata
@transaction.atomic
- def get_or_create_draft(self):
+ def get_or_create_draft(self, initial_version: str | None = None):
existing_draft = self.releases.filter(
status=CodebaseRelease.Status.DRAFT
).first()
if existing_draft:
return existing_draft
- draft_release = self.create_release()
+ # allow for overriding the version number of the initial draft
+ # this is currently used to create a less conflict-prone 0.0.1 draft when
+ # someone wants to import all their work with the github integration
+ if initial_version:
+ draft_release = self.create_release(version_number=initial_version)
+ else:
+ draft_release = self.create_release()
# reset fields that should not be copied over to a new draft
draft_release.doi = None
draft_release.release_notes = ""
@@ -1047,8 +1535,11 @@ def get_or_create_draft(self):
@transaction.atomic
def create_review_draft_from_release(self, source_release):
- # create a new "under review" release from an existing release
- # (should only be called from a publishable release)
+ """create a new "under review" release from an existing release
+
+ (should only be called from a publishable, non-external release)
+ """
+
source_id = source_release.id
review_draft = self.create_release(
status=CodebaseRelease.Status.UNDER_REVIEW, source_release=source_release
@@ -1065,6 +1556,8 @@ def create_release_from_source(self, source_release, release_metadata):
# set source_release.id to None to create a new release
# see https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances
source_release.id = None
+ source_release.imported_release_sync_state = None
+ source_release.git_ref_sync_state = None
source_release._state.adding = True
source_release.__dict__.update(**release_metadata)
source_release.save()
@@ -1123,7 +1616,7 @@ def create_release(
def save(self, rebuild_metadata=True, rebuild_release_metadata=True, **kwargs):
"""save the codebase and optionally rebuild metadata by updating codemeta_snapshot.
- If rebuild_release_metadata is True, all releases will be saved to trigger metadata updates
+ If rebuild_release_metadata is True, all internal releases will be saved to trigger metadata updates
"""
if rebuild_metadata and self.pk:
logger.debug("Building codemeta for codebase: %s", self)
@@ -1132,7 +1625,7 @@ def save(self, rebuild_metadata=True, rebuild_release_metadata=True, **kwargs):
# saving releases will trigger metadata rebuilding and updating
# the fs and git mirror if one exists
if rebuild_metadata and rebuild_release_metadata:
- for release in self.releases.all():
+ for release in self.releases.internal():
release.save(rebuild_metadata=True)
@classmethod
@@ -1231,6 +1724,14 @@ def with_codebase(self):
def with_submitter(self):
return self.prefetch_related("submitter")
+ def internal(self, **kwargs):
+ """returns only releases that are locally uploaded and not archived from an external service"""
+ return self.filter(imported_release_sync_state__isnull=True, **kwargs)
+
+ def imported(self, **kwargs):
+ """returns only releases that are imported from an external service such as GitHub"""
+ return self.filter(imported_release_sync_state__isnull=False, **kwargs)
+
def public(self, **kwargs):
return self.filter(status=CodebaseRelease.Status.PUBLISHED, **kwargs)
@@ -1320,6 +1821,19 @@ class Status(models.TextChoices):
max_length=32,
)
+ imported_release_sync_state = models.OneToOneField(
+ ImportedReleaseSyncState,
+ null=True,
+ on_delete=models.SET_NULL,
+ )
+ git_ref_sync_state = models.OneToOneField(
+ "library.GitRefSyncState",
+ null=True,
+ blank=True,
+ on_delete=models.SET_NULL,
+ related_name="release",
+ )
+
first_published_at = models.DateTimeField(null=True, blank=True)
last_published_on = models.DateTimeField(null=True, blank=True)
@@ -1496,6 +2010,10 @@ def get_notify_reviewers_of_changes_url(self):
},
)
+ def get_or_create_git_ref_sync_state(self):
+ """Return the git ref sync state for this release (creating if needed)."""
+ return GitRefSyncState.for_release(self)
+
def get_review(self):
review = getattr(self, "review", None)
return review if review and not review.closed else None
@@ -1563,27 +2081,21 @@ def validate_metadata(self):
return True
def validate_uploaded_files(self):
+ """Validates that this CodebaseRelease has the required files to be publish()-able
+ - at least one code file
+ - at least one documentation file
+ """
fs_api = self.get_fs_api()
- storage = fs_api.get_stage_storage(StagingDirectories.sip)
- code_msg = self.check_files(storage, FileCategoryDirectories.code)
- docs_msg = self.check_files(storage, FileCategoryDirectories.docs)
- msg = " ".join(m for m in [code_msg, docs_msg] if m)
- if msg:
- raise ValidationError(msg)
+ missing_categories = []
+ if not fs_api.check_category_file_exists(FileCategories.code):
+ missing_categories.append(FileCategories.code.name)
+ if not fs_api.check_category_file_exists(FileCategories.docs):
+ missing_categories.append(FileCategories.docs.name)
+ if missing_categories:
+ missing_str = " and ".join(missing_categories)
+ raise ValidationError(f"Must have at least one {missing_str} file.")
return True
- @staticmethod
- def check_files(storage, category: FileCategoryDirectories):
- # FIXME: document and/or refactor this API, should raise an exception or ...
- # currently returns an error message or ""
- uploaded_files = []
- if storage.exists(category.name):
- uploaded_files = list(storage.list(category))
- if uploaded_files:
- return ""
- else:
- return f"Must have at least one {category.name} file."
-
@property
def version_info(self):
if self.version_number:
@@ -1619,9 +2131,13 @@ def is_peer_review_requestable(self):
1. ready to be published
2. has not already been peer reviewed
3. a related PeerReview does not exist
+ 4. is not an imported, published release
"""
return (
- self.is_publishable and not self.peer_reviewed and self.get_review() is None
+ self.is_publishable
+ and not self.peer_reviewed
+ and self.get_review() is None
+ and not (self.is_imported and self.is_published)
)
@property
@@ -1858,12 +2374,19 @@ def get_status_color(self):
}
return COLOR_MAP.get(self.status)
+ @property
+ def is_imported(self):
+ return self.imported_release_sync_state is not None
+
def create_or_update_codemeta(self, force=True):
return self.get_fs_api().create_or_update_codemeta(force=force)
def get_fs_api(
self, mimetype_mismatch_message_level=MessageLevels.error
- ) -> CodebaseReleaseFsApi:
+ ) -> CodebaseReleaseFsApi | ImportedCodebaseReleaseFsApi:
+ """Factory method to return the appropriate filesystem API for this release."""
+ if self.is_imported:
+ return ImportedCodebaseReleaseFsApi.initialize(self)
return CodebaseReleaseFsApi.initialize(
self, mimetype_mismatch_message_level=mimetype_mismatch_message_level
)
@@ -1911,6 +2434,13 @@ def publish(self):
schedule_mint_public_doi(
self.id, dry_run=settings.DEPLOY_ENVIRONMENT.is_development
)
+ if GitRefSyncState.objects.filter(release__codebase=self.codebase).exists():
+ # if this codebase has a local git repo built by us, append the new release to it
+ from .tasks import append_releases_to_local_git_repo
+
+ transaction.on_commit(
+ lambda: append_releases_to_local_git_repo(self.codebase.id)
+ )
def _publish(self):
if not self.live:
@@ -1995,18 +2525,25 @@ def save(self, rebuild_metadata=True, defer_fs=True, **kwargs):
if not rebuild_metadata:
super().save(**kwargs)
else:
- logger.debug("Building codemeta for release: %s", self)
- old_codemeta = self.codemeta_snapshot
- self.codemeta_snapshot = self.codemeta.dict(serialize=True)
- super().save(**kwargs)
-
- if old_codemeta != self.codemeta_snapshot:
- if defer_fs:
- from .tasks import update_fs_release_metadata
-
- update_fs_release_metadata(self.id)
- else:
- self.get_fs_api().rebuild(metadata_only=True)
+ if not self.pk:
+ # save first so m2m/reverse relations used by codemeta generation can be accessed
+ # do not mess with the filesystem if this is a new release
+ super().save(**kwargs)
+ self.codemeta_snapshot = self.codemeta.dict(serialize=True)
+ super().save(update_fields=["codemeta_snapshot"])
+ else:
+ logger.debug("Building codemeta for release: %s", self)
+ old_codemeta = self.codemeta_snapshot
+ self.codemeta_snapshot = self.codemeta.dict(serialize=True)
+ super().save(**kwargs)
+
+ if old_codemeta != self.codemeta_snapshot:
+ if defer_fs:
+ from .tasks import update_fs_release_metadata
+
+ update_fs_release_metadata(self.id)
+ else:
+ self.get_fs_api().rebuild_metadata()
@classmethod
def get_indexed_objects(cls):
@@ -2778,9 +3315,16 @@ def editor_called_for_revisions(self, editor: MemberProfile):
review.status = ReviewStatus.AWAITING_AUTHOR_CHANGES
review.save()
recipients = {review.submitter.email, review.codebase_release.submitter.email}
+
+ if review.codebase_release.is_imported:
+ template_name = (
+ "library/review/email/model_revisions_requested_imported.jinja"
+ )
+ else:
+ template_name = "library/review/email/model_revisions_requested.jinja"
send_markdown_email(
subject="Peer review: revisions requested",
- template_name="library/review/email/model_revisions_requested.jinja",
+ template_name=template_name,
context={"review": review, "feedback": self},
to=recipients,
bcc=[settings.REVIEW_EDITOR_EMAIL],
@@ -2862,9 +3406,7 @@ def __init__(self, release: CodebaseRelease):
# raise ValueError("Invalid release with no License")
# FIXME: set all of these fields explicitly
- self.code_repository = (
- codebase.repository_url or "https://github.com/comses-model-library/"
- )
+ self.code_repository = codebase.code_repository
self.permanent_url = release.permanent_url
def convert_keywords(self):
@@ -3347,3 +3889,179 @@ def __str__(self):
HTTP Status: {self.http_status}, Message: {self.message},
Hash: {self.metadata_hash}, DOI: {self.doi}
"""
+
+
+class GitHubIntegrationFaqEntry(index.Indexed, Orderable):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="faq_entries",
+ on_delete=models.CASCADE,
+ )
+ question = models.CharField(max_length=500, help_text=_("The FAQ question"))
+ answer = MarkdownField(help_text=_("The FAQ answer in Markdown format"))
+
+ class Meta:
+ ordering = ["sort_order", "id"]
+
+
+class GitHubIntegrationInfoItemBase(index.Indexed, Orderable):
+ icon = models.CharField(
+ max_length=200,
+ help_text=_("optional icon class (e.g., 'fas fa-link')"),
+ blank=True,
+ )
+ text = MarkdownField(help_text=_("The info item text (Markdown)"))
+
+ panels = [
+ FieldPanel("icon"),
+ FieldPanel("text"),
+ ]
+
+ class Meta:
+ abstract = True
+ ordering = ["sort_order", "id"]
+
+
+class GitHubIntegrationExistingRepoInfoItem(GitHubIntegrationInfoItemBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="existing_repo_info_items",
+ on_delete=models.CASCADE,
+ )
+
+
+class GitHubIntegrationNewRepoInfoItem(GitHubIntegrationInfoItemBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="new_repo_info_items",
+ on_delete=models.CASCADE,
+ )
+
+
+class GitHubIntegrationTutorialVideoBase(index.Indexed, Orderable):
+ title = models.CharField(
+ blank=True, max_length=200, help_text=_("optional short title for the tutorial")
+ )
+ youtube_id = models.CharField(
+ max_length=32,
+ help_text=_("YouTube video id (e.g., dQw4w9WgXcQ)"),
+ )
+ caption = MarkdownField(blank=True, help_text=_("optional caption (Markdown)"))
+
+ panels = [
+ FieldPanel("title"),
+ FieldPanel("youtube_id"),
+ FieldPanel("caption"),
+ ]
+
+ class Meta:
+ abstract = True
+ ordering = ["sort_order", "id"]
+
+ def __str__(self):
+ if self.title:
+ return f"{self.title} ({self.youtube_id})"
+ return self.youtube_id
+
+
+class GitHubIntegrationExistingRepoTutorialVideo(GitHubIntegrationTutorialVideoBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="existing_repo_tutorial_videos",
+ on_delete=models.CASCADE,
+ )
+
+
+class GitHubIntegrationNewRepoTutorialVideo(GitHubIntegrationTutorialVideoBase):
+ configuration = ParentalKey(
+ "GitHubIntegrationConfiguration",
+ related_name="new_repo_tutorial_videos",
+ on_delete=models.CASCADE,
+ )
+
+
+@register_setting
+class GitHubIntegrationConfiguration(BaseSiteSetting, ClusterableModel):
+ is_beta = models.BooleanField(
+ default=True,
+ help_text=_("Mark the GitHub integration feature as a beta feature"),
+ )
+ is_enabled_globally = models.BooleanField(
+ default=False,
+ help_text=_(
+ "Enable GitHub integration for all users. When enabled, approved_users whitelist is ignored"
+ ),
+ )
+ approved_users = models.ManyToManyField(
+ settings.AUTH_USER_MODEL,
+ blank=True,
+ related_name="github_integration_approved",
+ help_text=_(
+ "Whitelist of users who can access GitHub integration. Superusers/staff are implicitly included"
+ ),
+ )
+
+ def can_use_github_integration(self, user):
+ """
+ Check if a user can use the GitHub integration feature.
+
+ returns True if:
+ - globally enabled (allows anonymous users)
+ - user is superuser or staff
+ - user is in the approved_users
+ """
+ if self.is_enabled_globally:
+ return True
+ if user is None or not user.is_authenticated:
+ return False
+ if user.is_superuser or user.is_staff:
+ return True
+ return self.approved_users.filter(id=user.id).exists()
+
+ @staticmethod
+ def github_app_settings_help_content():
+ """Create formatted content for a configuration help panel on the GitHub Integration settings page"""
+ info_html = format_html(
+ '[\w\-.]+)/releases", views.CodebaseReleaseViewSet
)
+router.register(
+ r"codebases/(?P[\w\-.]+)/git/remotes",
+ views.CodebaseGitRemoteViewSet,
+ basename="codebase-git-remotes",
+)
router.register(r"reviewers", views.PeerReviewerViewSet),
router.register(r"programming-languages", views.ProgrammingLanguageViewSet),
router.register(
@@ -47,6 +52,12 @@
)
urlpatterns = [
+ path("github-sync-webhook/", views.github_sync_webhook, name="github-sync-webhook"),
+ path(
+ "github/",
+ views.GitHubSyncOverviewView.as_view(),
+ name="github-sync-overview",
+ ),
path(
"reviews/",
TemplateView.as_view(template_name="library/review/peer-review-overview.jinja"),
diff --git a/django/library/views.py b/django/library/views.py
index 909fb323e..26888e31a 100644
--- a/django/library/views.py
+++ b/django/library/views.py
@@ -1,15 +1,23 @@
+import hashlib
+import hmac
import json
+from allauth.socialaccount.models import SocialAccount
from django.forms import Form
+from django.conf import settings
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin
from django.core.exceptions import PermissionDenied
-from django.db import transaction
+from django.db import transaction, IntegrityError
+from django.db.models import Q
from django.http import HttpResponse, Http404, HttpResponseRedirect
from django.shortcuts import get_object_or_404, redirect
from django.urls import resolve, reverse
+from django.utils import timezone
from django.utils.translation import gettext_lazy as _
+from django.utils.text import slugify
from django.views import View
-from django.views.generic import FormView
+from django.views.decorators.csrf import csrf_exempt
+from django.views.generic import FormView, TemplateView
from django.views.generic.base import RedirectView
from django.views.generic.detail import DetailView
from django.views.generic.edit import UpdateView
@@ -27,6 +35,7 @@
from rest_framework.exceptions import (
PermissionDenied as DrfPermissionDenied,
ValidationError,
+ NotFound,
)
from rest_framework.filters import OrderingFilter
from rest_framework.permissions import IsAuthenticatedOrReadOnly
@@ -45,7 +54,14 @@
HtmlNoDeleteViewSet,
)
from core.pagination import SmallResultSetPagination
-from core.serializers import RelatedMemberProfileSerializer
+from .github_integration import (
+ GitHubReleaseImporter,
+ GitHubRepoValidator,
+ get_github_installation_status,
+ list_github_releases_for_remote,
+ GitHubApi,
+ extract_semver,
+)
from .forms import (
PeerReviewerFeedbackReviewerForm,
PeerReviewInvitationReplyForm,
@@ -53,12 +69,18 @@
PeerReviewerFeedbackEditorForm,
PeerReviewFilterForm,
)
-from .fs import FileCategoryDirectories, StagingDirectories, MessageLevels
+from .fs import (
+ FileCategories,
+ StagingDirectories,
+ MessageLevels,
+)
from .models import (
Codebase,
+ CodebaseGitRemote,
CodebaseRelease,
Contributor,
CodebaseImage,
+ GithubIntegrationAppInstallation,
License,
ProgrammingLanguage,
PeerReview,
@@ -66,11 +88,20 @@
PeerReviewerFeedback,
PeerReviewInvitation,
ReviewStatus,
+ GitHubIntegrationConfiguration,
+)
+from wagtail.models import Site
+from .models import GitRefSyncState, ImportedReleaseSyncState
+from .permissions import (
+ CodebaseReleaseUnpublishedFilePermissions,
+ GitHubIntegrationPermission,
)
-from .permissions import CodebaseReleaseUnpublishedFilePermissions
from .serializers import (
+ CodebaseGitRemoteSerializer,
CodebaseSerializer,
CodebaseReleaseSerializer,
+ CodebaseReleaseWithGitRefSyncStateSerializer,
+ RelatedCodebaseSerializer,
ContributorSerializer,
DownloadRequestSerializer,
PeerReviewerSerializer,
@@ -82,9 +113,15 @@
PeerReviewFeedbackEditorSerializer,
PeerReviewEventLogSerializer,
)
+from .tasks import (
+ import_github_release_task,
+ push_all_releases_to_github,
+ build_local_git_repo,
+)
import logging
import pathlib
+from packaging.version import Version
logger = logging.getLogger(__name__)
@@ -492,6 +529,28 @@ def list(self, request, *args, **kwargs):
serializer = self.get_serializer(queryset, many=True)
return Response(serializer.data)
+ @action(
+ detail=False, methods=["get"], permission_classes=[permissions.IsAuthenticated]
+ )
+ def submitted_codebases(self, request, *args, **kwargs):
+ """
+ Returns a list of codebases submitted by the requesting user
+ """
+ queryset = (
+ self.get_queryset().filter(submitter=request.user).order_by("-date_created")
+ )
+ serializer = RelatedCodebaseSerializer(
+ queryset,
+ many=True,
+ context=self.get_serializer_context(),
+ )
+ return Response(serializer.data)
+
+ @action(detail=False, methods=["get"])
+ def github_installation_status(self, request, *args, **kwargs):
+ data = get_github_installation_status(request.user)
+ return Response(data=data, status=status.HTTP_200_OK)
+
def get_queryset(self):
if self.action == "list":
return self.queryset.public()
@@ -501,7 +560,8 @@ def get_queryset(self):
def perform_create(self, serializer):
super().perform_create(serializer)
codebase = serializer.instance
- return codebase.get_or_create_draft()
+ initial_version = self.request.query_params.get("initial_version")
+ return codebase.get_or_create_draft(initial_version=initial_version)
def retrieve(self, request, *args, **kwargs):
instance = self.get_object()
@@ -568,18 +628,379 @@ def get_redirect_url(self, *args, **kwargs):
return super().get_redirect_url(*args, **kwargs)
-class CodebaseImagePermission(permissions.BasePermission):
+class CanChangeCodebase(permissions.BasePermission):
def has_permission(self, request, view):
codebase = get_object_or_404(Codebase, identifier=view.kwargs["identifier"])
return request.user.has_perm("library.change_codebase", codebase)
+class CodebaseGitRemoteViewSet(
+ mixins.RetrieveModelMixin, mixins.UpdateModelMixin, viewsets.GenericViewSet
+):
+ queryset = CodebaseGitRemote.objects.all()
+ permission_classes = (CanChangeCodebase, GitHubIntegrationPermission)
+ pagination_class = None
+ serializer_class = CodebaseGitRemoteSerializer
+
+ def get_queryset(self):
+ resolved = resolve(self.request.path)
+ identifier = resolved.kwargs["identifier"]
+ return self.queryset.filter(codebase__identifier=identifier).prefetch_related(
+ "git_ref_sync_states", "imported_sync_states"
+ )
+
+ def get_codebase(self):
+ try:
+ return Codebase.objects.get(identifier=self.kwargs.get("identifier"))
+ except Codebase.DoesNotExist:
+ raise NotFound("Codebase not found.")
+
+ def _forward_integrity_error(self, e: IntegrityError):
+ msg = str(e)
+ if "single_active_remote" in msg:
+ raise ValidationError(
+ "There can only be one active repository for a codebase at a time."
+ )
+ else:
+ raise ValidationError(
+ "This repository is already connected to another model."
+ )
+
+ def update(self, request, *args, **kwargs):
+ try:
+ return super().update(request, *args, **kwargs)
+ except IntegrityError as e:
+ self._forward_integrity_error(e)
+
+ def list(self, request, *args, **kwargs):
+ queryset = self.get_queryset()
+ if request.accepted_renderer.format == "html":
+ context = {
+ "codebase": self.get_codebase(),
+ "remotes": queryset,
+ }
+ return Response(
+ context,
+ template_name="library/codebases/git.jinja",
+ )
+ serializer = self.get_serializer(queryset, many=True)
+ return Response(serializer.data)
+
+ @action(detail=False, methods=["post"])
+ def push_all(self, request, *args, **kwargs):
+ """Manually push all releases to GitHub for the active remote.
+
+ Immediately marks each release's GitRefSyncState as RUNNING for UI feedback,
+ then enqueues an async task to perform the push and create GitHub releases.
+ """
+ codebase = self.get_codebase()
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ raise ValidationError("Active remote not found")
+
+ # prevent concurrent push jobs for the same codebase
+ if GitRefSyncState.objects.filter(
+ Q(release__codebase=codebase) | Q(codebase=codebase),
+ status=GitRefSyncState.Status.RUNNING,
+ ).exists():
+ raise ValidationError(
+ "Push already in progress for this codebase. Please wait."
+ )
+
+ # start all push jobs for this remote (set remotes and mark running)
+ started = GitRefSyncState.start_all_push_jobs_for_codebase(
+ codebase, active_remote
+ )
+ if not started:
+ return Response(status=status.HTTP_204_NO_CONTENT)
+ # enqueue task to carry out all the push jobs we just started
+ push_all_releases_to_github(codebase.id, active_remote.id)
+ return Response(status=status.HTTP_202_ACCEPTED, data="Push started.")
+
+ @action(detail=False, methods=["get"])
+ def active_remote(self, request, *args, **kwargs):
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if active_remote:
+ serializer = self.get_serializer(active_remote)
+ return Response(data=serializer.data, status=status.HTTP_200_OK)
+ return Response(data=None, status=status.HTTP_200_OK)
+
+ @action(detail=False, methods=["get"])
+ def local_releases(self, request, *args, **kwargs):
+ """Return all local releases for this codebase with git ref sync states."""
+ codebase = self.get_codebase()
+ qs = CodebaseRelease.objects.filter(codebase=codebase).select_related(
+ "codebase", "git_ref_sync_state"
+ )
+ releases = list(qs)
+ releases.sort(key=lambda r: Version(r.version_number), reverse=True)
+ serializer = CodebaseReleaseWithGitRefSyncStateSerializer(
+ releases, many=True, context=self.get_serializer_context()
+ )
+ return Response(serializer.data)
+
+ @action(detail=False, methods=["get"])
+ def github_releases(self, request, *args, **kwargs):
+ """fetch releases from the connected GitHub repository for this codebase
+
+ requires an active remote. returns minimal release info suitable for client display
+ and import selection.
+ """
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ return Response([], status=status.HTTP_200_OK)
+ try:
+ releases = list_github_releases_for_remote(active_remote)
+ return Response(releases, status=status.HTTP_200_OK)
+ except Exception as e:
+ logger.exception("Failed to fetch GitHub releases: %s", e)
+ raise ValidationError(
+ "Unable to fetch GitHub releases for the active remote"
+ )
+
+ @action(detail=False, methods=["post"])
+ def import_github_release(self, request, *args, **kwargs):
+ """Start an async import for a specific GitHub release id for the active remote.
+
+ body params:
+ - github_release_id: str | int (required)
+ - custom_version: str (optional) when the release tag/name has no semver
+ """
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ raise ValidationError("Active remote not found")
+ github_release_id = request.data.get("github_release_id")
+ if not github_release_id:
+ raise ValidationError("'github_release_id' is required")
+ custom_version = request.data.get("custom_version")
+ if custom_version is not None:
+ if not isinstance(custom_version, str):
+ raise ValidationError(
+ "'custom_version' must be a semantic version like X.Y.Z"
+ )
+ candidate = custom_version.strip()
+ parsed_version = extract_semver(candidate)
+ is_exact_semver = parsed_version and (
+ candidate == parsed_version or candidate == f"v{parsed_version}"
+ )
+ if not is_exact_semver:
+ raise ValidationError(
+ "'custom_version' must be a semantic version like X.Y.Z"
+ )
+ custom_version = parsed_version
+ # require existing sync state
+ sync_state = ImportedReleaseSyncState.objects.filter(
+ remote=active_remote, github_release_id=str(github_release_id)
+ ).first()
+ if not sync_state:
+ raise ValidationError(
+ "Import not prepared. refresh GitHub releases and try again."
+ )
+ if sync_state.status == ImportedReleaseSyncState.Status.RUNNING:
+ raise ValidationError("GitHub release import is already in progress")
+ # mark as RUNNING and dispatch import task
+ sync_state.mark_running()
+ import_github_release_task(
+ active_remote.codebase_id,
+ active_remote.id,
+ str(github_release_id),
+ custom_version,
+ )
+ return Response(status=status.HTTP_202_ACCEPTED, data="Import started.")
+
+ @action(detail=False, methods=["get"])
+ def submitter_installation_status(self, request, *args, **kwargs):
+ codebase = self.get_codebase()
+ submitter = codebase.submitter
+ data = get_github_installation_status(submitter)
+ return Response(data=data, status=status.HTTP_200_OK)
+
+ @action(detail=False, methods=["post"])
+ def setup_user_github_remote(self, request, *args, **kwargs):
+ codebase = self.get_codebase()
+ installation = codebase.submitter.github_integration_app_installation
+ if not installation:
+ raise ValidationError(
+ "Installation of the Github integration app is required"
+ )
+
+ is_preexisting = bool(request.data.get("is_preexisting", False))
+ if not codebase.live and not is_preexisting:
+ raise ValidationError("This model does not have any published releases")
+
+ repo_name = request.data.get("repo_name")
+ if not repo_name:
+ raise ValidationError("Repository name is required")
+
+ validator = GitHubRepoValidator(repo_name)
+ try:
+ validator.validate_format()
+ repo_html_url = validator.get_url_for_connectable_user_repo(
+ installation, is_preexisting
+ )
+ except ValueError as e:
+ error_message = e.args[0] if e.args else "Invalid repository configuration"
+ raise ValidationError(error_message)
+
+ # swap out a new active remote for the old one
+ queryset = self.get_queryset()
+ current_remote = queryset.filter(is_active=True).first()
+ try:
+ with transaction.atomic():
+ if current_remote:
+ current_remote.is_active = False
+ current_remote.save(update_fields=["is_active", "last_modified"])
+ new_remote = codebase.create_remote(
+ owner=installation.github_login,
+ repo_name=repo_name,
+ is_user_repo=True,
+ is_preexisting=is_preexisting,
+ is_active=True,
+ url=repo_html_url,
+ )
+ # check if the refs were previously bound to a different remote and reassign
+ # this handles both: switching from an active remote, and reconnecting after
+ # a manual disconnect (where the refs still point to the old remote)
+ previous_remote = GitRefSyncState.get_last_remote_for_codebase(codebase)
+ if previous_remote and previous_remote.id != new_remote.id:
+ GitRefSyncState.reassign_remotes_for_codebase(
+ codebase=codebase, remote=new_remote
+ )
+ except IntegrityError as e:
+ self._forward_integrity_error(e)
+ return Response(
+ status=status.HTTP_200_OK,
+ data="Repository connection successful.",
+ )
+
+ @action(detail=False, methods=["post"])
+ def disconnect_remote(self, request, *args, **kwargs):
+ """Deactivate the current active remote without reassigning refs.
+
+ This allows users to disconnect without losing push state, so they can
+ reconnect to the same remote later if desired.
+ """
+ queryset = self.get_queryset()
+ active_remote = queryset.filter(is_active=True).first()
+ if not active_remote:
+ raise ValidationError("No active remote to disconnect")
+
+ active_remote.is_active = False
+ active_remote.save(update_fields=["is_active", "last_modified"])
+ return Response(
+ status=status.HTTP_200_OK,
+ data="Repository disconnected.",
+ )
+
+ @action(detail=False, methods=["post"])
+ def build_local_repo(self, request, *args, **kwargs):
+ """Build or update the local git mirror for this codebase without pushing."""
+ codebase = self.get_codebase()
+ build_local_git_repo(codebase.id)
+ return Response(
+ status=status.HTTP_202_ACCEPTED, data="Local repository build started."
+ )
+
+
+class GitHubSyncOverviewView(TemplateView):
+ """View for GitHub integration overview page with permission check."""
+
+ template_name = "library/github_overview.jinja"
+
+ def dispatch(self, request, *args, **kwargs):
+ try:
+ site = Site.objects.get(is_default_site=True)
+ config = GitHubIntegrationConfiguration.for_site(site)
+ except (Site.DoesNotExist, GitHubIntegrationConfiguration.DoesNotExist):
+ raise PermissionDenied("GitHub integration is not available")
+
+ if not config.can_use_github_integration(request.user):
+ raise PermissionDenied("GitHub integration is not available")
+ return super().dispatch(request, *args, **kwargs)
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+ context["github_model_library_org_name"] = (
+ settings.GITHUB_MODEL_LIBRARY_ORG_NAME
+ )
+ return context
+
+
+@csrf_exempt
+def github_sync_webhook(request):
+ """
+ Handle GitHub app webhook events:
+ - new installations (match to connected socialaccount and save the installation record)
+
+ for local testing, use smee.io or similar to forward webhooks:
+ https://docs.github.com/en/webhooks/using-webhooks/handling-webhook-deliveries#setup
+ """
+ # verify the request signature
+ signature_header = request.headers.get("X-Hub-Signature-256")
+ if not signature_header:
+ return HttpResponse("Missing signature header", status=403)
+ secret = settings.GITHUB_INTEGRATION_APP_WEBHOOK_SECRET
+ hash_object = hmac.new(
+ secret.encode("utf-8"),
+ msg=request.body,
+ digestmod=hashlib.sha256,
+ )
+ expected_signature = f"sha256={hash_object.hexdigest()}"
+ if not hmac.compare_digest(expected_signature, signature_header):
+ return HttpResponse("Invalid signature", status=403)
+
+ # parse the event
+ event = request.headers.get("X-GitHub-Event")
+ if event == "installation":
+ try:
+ payload = json.loads(request.body)
+ logger.debug("received installation webhook %s", payload)
+ action = payload["action"]
+ sender = payload["sender"]
+ installation_id = payload["installation"]["id"]
+ sender_login = sender["login"]
+ except (json.JSONDecodeError, KeyError, TypeError):
+ return HttpResponse("Malformed payload", status=400)
+ # match based on the uid
+ uid = sender["id"]
+ social_account = SocialAccount.objects.filter(
+ provider="github", uid=str(uid)
+ ).first()
+ if not social_account:
+ logger.warning(
+ "No matching social account found for installation event: %s", payload
+ )
+ return HttpResponse(status=404)
+ if action == "deleted":
+ installation = GithubIntegrationAppInstallation.objects.filter(
+ user=social_account.user,
+ ).delete()
+ return HttpResponse("OK", status=200)
+ else:
+ installation = GithubIntegrationAppInstallation.objects.update_or_create(
+ user=social_account.user,
+ defaults={
+ "github_user_id": uid,
+ "github_login": sender_login,
+ "installation_id": installation_id,
+ },
+ )
+ logger.debug("Installation record updated/created: %s", installation)
+ return HttpResponse("OK", status=200)
+
+ return HttpResponse(status=202)
+
+
class CodebaseImageViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
lookup_field = "codebaseimage_id"
lookup_value_regex = r"\d+"
queryset = CodebaseImage.objects.all()
serializer_class = CodebaseImageSerializer
- permission_classes = (CodebaseImagePermission,)
+ permission_classes = (CanChangeCodebase,)
def get_queryset(self):
resolved = resolve(self.request.path)
@@ -885,7 +1306,7 @@ def notify_reviewers_of_changes(self, request, **kwargs):
@transaction.atomic
def request_peer_review(self, request, identifier, version_number):
"""
- If a peer review is requestable (publishable, not reviewed, no related review exists):
+ If a peer review is requestable (publishable, not reviewed, no related review exists, and not imported+published):
- Create a new "under review" draft release if the release from which the request was made is published
- Otherwise, update the existing draft release to be "under review"
- Create a new peer review object and send an email to the author
@@ -895,25 +1316,40 @@ def request_peer_review(self, request, identifier, version_number):
codebase__identifier=identifier,
version_number=version_number,
)
+ if codebase_release.is_imported and codebase_release.is_published:
+ raise ValidationError(
+ "Cannot request a peer review for an imported release that has already been published. As a workaround, you can make another release on GitHub and request a peer review for that."
+ )
codebase_release.validate_publishable()
existing_review = PeerReview.get_codebase_latest_active_review(
codebase_release.codebase
)
+ # if there is an existing active review, simply redirect to that release
if existing_review:
review = existing_review
review_release = existing_review.codebase_release
created = False
+ # if not create a new review
else:
- if codebase_release.is_draft:
- codebase_release.status = CodebaseRelease.Status.UNDER_REVIEW
- codebase_release.save(update_fields=["status"])
- review_release = codebase_release
- else:
+ # first, check if this release status is already under or completed review
+ # if so, something went wrong so we'll
+ if codebase_release.is_under_review or codebase_release.is_review_complete:
+ raise ValidationError(
+ "Cannot re-request a review on a release that has already completed or is undergoing review."
+ )
+ # if the release is published, make a new draft copy
+ elif codebase_release.is_published:
review_release = (
codebase_release.codebase.create_review_draft_from_release(
codebase_release
)
)
+ # if the release is a draft/unpublished, change the status
+ else:
+ codebase_release.status = CodebaseRelease.Status.UNDER_REVIEW
+ codebase_release.save(update_fields=["status"])
+ review_release = codebase_release
+
review = PeerReview.objects.create(
codebase_release=review_release,
submitter=request.user.member_profile,
@@ -1027,7 +1463,7 @@ def get_url_matcher(cls):
r"codebases/(?P[\w\-.]+)",
r"/releases/(?P\d+\.\d+\.\d+)",
r"/files/{}/(?P{})".format(
- cls.stage.name, "|".join(c.name for c in FileCategoryDirectories)
+ cls.stage.name, "|".join(c.name for c in FileCategories)
),
]
)
@@ -1038,20 +1474,20 @@ def get_queryset(self):
queryset = self.queryset.filter(codebase__identifier=identifier)
return queryset.accessible(user=self.request.user)
- def get_category(self) -> FileCategoryDirectories:
+ def get_list_url(self, api):
+ raise NotImplementedError
+
+ def get_category(self) -> FileCategories:
category = self.get_parser_context(self.request)["kwargs"]["category"]
try:
- return FileCategoryDirectories[category]
+ return FileCategories[category]
except KeyError:
raise ValidationError(
- "Target folder name {} invalid. Must be one of {}".format(
- category, list(d.name for d in FileCategoryDirectories)
+ "Target category name {} invalid. Must be one of {}".format(
+ category, list(d.name for d in FileCategories)
)
)
- def get_list_url(self, api):
- raise NotImplementedError
-
def list(self, request, *args, **kwargs):
codebase_release = self.get_object()
api = codebase_release.get_fs_api()
@@ -1090,6 +1526,35 @@ class CodebaseReleaseFilesSipViewSet(BaseCodebaseReleaseFilesViewSet):
def get_list_url(self, api):
return api.get_sip_list_url
+ @action(detail=False, methods=["post"])
+ def update_category(self, request, **kwargs):
+ """update a file's category, currently only for imported releases
+
+ Note: the category given in the request data is the new category for the file
+ and the category in the URL is the current category of the file (or anything,
+ it is ignored here)
+ """
+ codebase_release = self.get_object()
+ if not codebase_release.is_imported:
+ raise ValidationError("Cannot update file category on non-imported release")
+ fs_api = codebase_release.get_fs_api()
+ file_path = request.data.get("path")
+ new_category_str = request.data.get("category")
+ if not file_path or not new_category_str:
+ raise ValidationError("Both a file 'path' and 'category' are required")
+ try:
+ new_category = FileCategories[new_category_str]
+ except KeyError:
+ raise ValidationError(
+ f"Target category name {new_category_str} invalid. Must be one of {[c.name for c in FileCategories]}"
+ )
+ try:
+ fs_api.manifest.update_file_category(file_path, new_category)
+ except ValueError as e:
+ error_message = e.args[0] if e.args else "Unable to update file category"
+ raise ValidationError(error_message)
+ return Response(status=status.HTTP_200_OK)
+
class CodebaseReleaseFilesOriginalsViewSet(BaseCodebaseReleaseFilesViewSet):
stage = StagingDirectories.originals
diff --git a/django/requirements.txt b/django/requirements.txt
index 09d4f6690..16ba024ca 100644
--- a/django/requirements.txt
+++ b/django/requirements.txt
@@ -27,6 +27,7 @@ Django==5.2.11
drf-spectacular==0.29.0
elasticsearch-dsl>=7.0.0,<8.0.0
elasticsearch>=7.0.0,<8.0.0
+gitpython==3.1.50
html2text>=2016.9.19
huey==2.6.0
jinja2==3.1.6
@@ -37,6 +38,7 @@ numpy==1.26.4
pandas==2.2.2
psycopg2-binary==2.9.11
pyjwt[crypto]>=2.4.0,<3.0.0
+PyGithub==2.5.0
pytz==2025.2
pyyaml>=6.0.1
# used for institution -> affiliation data migration
@@ -56,6 +58,5 @@ wagtail-modeladmin==2.2.0
invoke
ipython
pgcli
-# replace with pypi scicodes install eventually
git+https://github.com/comses/codemeticulous.git
sqlparse>=0.5.4 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/frontend/src/apps/codebase_edit.ts b/frontend/src/apps/codebase_edit.ts
index 021a76880..c31bb185c 100644
--- a/frontend/src/apps/codebase_edit.ts
+++ b/frontend/src/apps/codebase_edit.ts
@@ -4,5 +4,5 @@ import { createApp } from "vue";
import CodebaseEditForm from "@/components/CodebaseEditForm.vue";
import { extractDataParams } from "@/util";
-const props = extractDataParams("codebase-form", ["identifier"]);
+const props = extractDataParams("codebase-form", ["identifier", "canUseGithubIntegration"]);
createApp(CodebaseEditForm, props).mount("#codebase-form");
diff --git a/frontend/src/apps/github_config.ts b/frontend/src/apps/github_config.ts
new file mode 100644
index 000000000..3b4d909be
--- /dev/null
+++ b/frontend/src/apps/github_config.ts
@@ -0,0 +1,12 @@
+import "vite/modulepreload-polyfill";
+
+import { createApp } from "vue";
+import { extractDataParams } from "@/util";
+import GitHubIntegrationConfiguration from "@/components/GitHubIntegrationConfiguration.vue";
+
+const props = extractDataParams("github-config", [
+ "codebaseIdentifier",
+ "defaultRepoName",
+ "isCodebaseLive",
+]);
+createApp(GitHubIntegrationConfiguration, props).mount("#github-config");
diff --git a/frontend/src/apps/github_overview.ts b/frontend/src/apps/github_overview.ts
new file mode 100644
index 000000000..0293c5dc8
--- /dev/null
+++ b/frontend/src/apps/github_overview.ts
@@ -0,0 +1,6 @@
+import "vite/modulepreload-polyfill";
+
+import { createApp } from "vue";
+import GitHubIntegrationOverview from "@/components/GitHubIntegrationOverview.vue";
+
+createApp(GitHubIntegrationOverview).mount("#github-overview");
diff --git a/frontend/src/apps/release_editor.ts b/frontend/src/apps/release_editor.ts
index 8b479a2a9..28cd4585c 100644
--- a/frontend/src/apps/release_editor.ts
+++ b/frontend/src/apps/release_editor.ts
@@ -6,6 +6,7 @@ import { createRouter, createWebHashHistory } from "vue-router";
import App from "@/components/releaseEditor/App.vue";
import MetadataFormPage from "@/components/releaseEditor/MetadataFormPage.vue";
import UploadFormPage from "@/components/releaseEditor/UploadFormPage.vue";
+import ImportedArchivePage from "@/components/releaseEditor/ImportedArchivePage.vue";
import ContributorsPage from "@/components/releaseEditor/ContributorsPage.vue";
import { extractDataParams } from "@/util";
@@ -15,6 +16,7 @@ const props = extractDataParams("release-editor", [
"reviewStatus",
"isLive",
"canEditOriginals",
+ "isImported",
]);
// check if ?publish or ?upload-image is in the url, set prop, and clear from the url
@@ -26,14 +28,21 @@ window.history.replaceState({}, "", window.location.pathname + window.location.h
const app = createApp(App, props);
const pinia = createPinia();
+const filesRouteName = props.isImported ? "package" : "upload";
+const filesRouteComponent = props.isImported ? ImportedArchivePage : UploadFormPage;
+
const router = createRouter({
history: createWebHashHistory(),
routes: [
- // only include upload route when original files are editable
- { path: "/", redirect: { name: props.canEditOriginals ? "upload" : "metadata" } },
- ...(props.canEditOriginals
- ? [{ path: "/upload", component: UploadFormPage, name: "upload" }]
- : []),
+ // only default to upload route when original files are editable
+ {
+ path: "/",
+ redirect: {
+ name: props.canEditOriginals ? filesRouteName : "metadata",
+ },
+ },
+ // use the imported archive page for imported releases
+ { path: `/${filesRouteName}`, component: filesRouteComponent, name: filesRouteName },
{ path: "/metadata", component: MetadataFormPage, name: "metadata" },
{ path: "/contributors", component: ContributorsPage, name: "contributors" },
],
diff --git a/frontend/src/components/CodebaseEditForm.vue b/frontend/src/components/CodebaseEditForm.vue
index 583b2e6e3..644f298f2 100644
--- a/frontend/src/components/CodebaseEditForm.vue
+++ b/frontend/src/components/CodebaseEditForm.vue
@@ -48,27 +48,36 @@
help="Add tags to categorize your model and make it more discoverable. Press enter after entering each tag."
/>
+ Please note that this model release was imported from GitHub, so the file structure may differ from + a typical release package in the CoMSES Model Library. +
+ {% endif %}See the infra wiki for configuration info.
' + ) + app_name = settings.GITHUB_INTEGRATION_APP_NAME + org_name = settings.GITHUB_MODEL_LIBRARY_ORG_NAME + warnings = [] + if not app_name: + warnings.append( + "GitHub app name is not configured. Set GITHUB_INTEGRATION_APP_NAME in the environment." + ) + if not org_name: + warnings.append( + "GitHub organization name is not configured. Set GITHUB_MODEL_LIBRARY_ORG_NAME in the environment." + ) + if warnings: + help_html = format_html( + 'WARNING! {}
',
+ " ".join(warnings),
+ )
+ else:
+ url = f"https://github.com/organizations/{org_name}/settings/apps/{slugify(app_name)}"
+ help_html = format_html(
+ '{} settings on GitHub',
+ url,
+ app_name,
+ )
+ return info_html + help_html
+
+ panels = [
+ HelpPanel(content=github_app_settings_help_content()),
+ FieldPanel("is_beta"),
+ FieldPanel("is_enabled_globally"),
+ FieldPanel(
+ "approved_users",
+ widget=FilteredSelectMultiple("approved users", is_stacked=False),
+ ),
+ InlinePanel("faq_entries", label="FAQ Entries"),
+ InlinePanel("existing_repo_info_items", label="Existing Repo Info Items"),
+ InlinePanel(
+ "existing_repo_tutorial_videos", label="Existing Repo Tutorial Videos"
+ ),
+ InlinePanel("new_repo_info_items", label="New Repo Info Items"),
+ InlinePanel("new_repo_tutorial_videos", label="New Repo Tutorial Videos"),
+ ]
diff --git a/django/library/permissions.py b/django/library/permissions.py
index 51d9675a0..4dede205e 100644
--- a/django/library/permissions.py
+++ b/django/library/permissions.py
@@ -1,6 +1,24 @@
-from rest_framework.permissions import DjangoModelPermissions
+from rest_framework.permissions import DjangoModelPermissions, BasePermission
+from rest_framework.exceptions import PermissionDenied as DrfPermissionDenied
+from wagtail.models import Site
from core.permissions import ObjectPermissions
+from library.models import GitHubIntegrationConfiguration
+
+
+class GitHubIntegrationPermission(BasePermission):
+ """Permission class to check if user can use GitHub integration."""
+
+ def has_permission(self, request, view):
+ try:
+ site = Site.objects.get(is_default_site=True)
+ config = GitHubIntegrationConfiguration.for_site(site)
+ except (Site.DoesNotExist, GitHubIntegrationConfiguration.DoesNotExist):
+ raise DrfPermissionDenied("GitHub integration is not available")
+
+ if not config.can_use_github_integration(request.user):
+ raise DrfPermissionDenied("GitHub integration is not available")
+ return True
class CodebaseReleaseUnpublishedFilePermissions(ObjectPermissions):
diff --git a/django/library/serializers.py b/django/library/serializers.py
index 43b493f5e..75efb470a 100644
--- a/django/library/serializers.py
+++ b/django/library/serializers.py
@@ -30,6 +30,9 @@
RelatedUserSerializer,
)
from .models import (
+ CodebaseGitRemote,
+ GitRefSyncState,
+ ImportedReleaseSyncState,
PeerReviewer,
ReleaseContributor,
Codebase,
@@ -466,6 +469,9 @@ class RelatedCodebaseSerializer(serializers.ModelSerializer, FeaturedImageMixin)
Sparse codebase serializer
"""
+ absolute_url = serializers.URLField(source="get_absolute_url", read_only=True)
+ github_config_url = serializers.SerializerMethodField(read_only=True)
+ active_git_remote = serializers.SerializerMethodField(read_only=True)
all_contributors = ContributorSerializer(many=True, read_only=True)
tags = TagSerializer(many=True)
version_number = serializers.ReadOnlyField(source="latest_version.version_number")
@@ -479,6 +485,15 @@ class RelatedCodebaseSerializer(serializers.ModelSerializer, FeaturedImageMixin)
live = serializers.ReadOnlyField()
description = MarkdownField()
+ def get_github_config_url(self, instance):
+ return reverse("library:codebase-git-remotes-list", args=[instance.identifier])
+
+ def get_active_git_remote(self, instance):
+ active_git_remote = instance.active_git_remote
+ if active_git_remote:
+ return CodebaseGitRemoteSerializer(active_git_remote).data
+ return None
+
def create(self, validated_data):
return create(self.Meta.model, validated_data, self.context)
@@ -488,7 +503,10 @@ def update(self, instance, validated_data):
class Meta:
model = Codebase
fields = (
+ "absolute_url",
"all_contributors",
+ "github_config_url",
+ "active_git_remote",
"tags",
"title",
"first_published_at",
@@ -504,6 +522,53 @@ class Meta:
)
+class GitRefSyncStateSerializer(serializers.ModelSerializer):
+ can_push = serializers.SerializerMethodField()
+
+ def get_can_push(self, instance):
+ return instance.can_push()
+
+ class Meta:
+ model = GitRefSyncState
+ fields = "__all__"
+
+
+class ImportedReleaseSyncStateSerializer(serializers.ModelSerializer):
+ can_reimport = serializers.SerializerMethodField()
+
+ def get_can_reimport(self, instance):
+ return instance.can_reimport()
+
+ class Meta:
+ model = ImportedReleaseSyncState
+ fields = "__all__"
+
+
+class CodebaseGitRemoteSerializer(serializers.ModelSerializer):
+ is_active = serializers.ReadOnlyField()
+
+ class Meta:
+ model = CodebaseGitRemote
+ fields = (
+ "id",
+ "owner",
+ "repo_name",
+ "url",
+ "is_user_repo",
+ "is_preexisting",
+ "is_active",
+ )
+ read_only_fields = (
+ "id",
+ "owner",
+ "repo_name",
+ "url",
+ "is_user_repo",
+ "is_preexisting",
+ "is_active",
+ )
+
+
class CodebaseImageSerializer(serializers.ModelSerializer):
identifier = serializers.IntegerField(source="id")
name = serializers.CharField(source="title")
@@ -596,6 +661,7 @@ class CodebaseReleaseSerializer(serializers.ModelSerializer):
release_notes = MarkdownField(max_length=2048)
urls = serializers.SerializerMethodField()
review_status = serializers.SerializerMethodField()
+ imported_release_sync_state = ImportedReleaseSyncStateSerializer(read_only=True)
def get_urls(self, instance):
request_peer_review_url = instance.get_request_peer_review_url()
@@ -617,6 +683,7 @@ class Meta:
model = CodebaseRelease
fields = (
"absolute_url",
+ "status",
"can_edit_originals",
"citation_text",
"release_contributors",
@@ -645,11 +712,21 @@ class Meta:
"output_data_url",
"version_number",
"id",
+ "imported_release_sync_state",
"share_url",
"urls",
)
+class CodebaseReleaseWithGitRefSyncStateSerializer(CodebaseReleaseSerializer):
+ """Extends base release serializer to include git ref sync state for local mirror context"""
+
+ git_ref_sync_state = GitRefSyncStateSerializer(read_only=True)
+
+ class Meta(CodebaseReleaseSerializer.Meta):
+ fields = CodebaseReleaseSerializer.Meta.fields + ("git_ref_sync_state",)
+
+
class CodebaseReleaseEditSerializer(CodebaseReleaseSerializer):
possible_licenses = serializers.SerializerMethodField()
diff --git a/django/library/tasks.py b/django/library/tasks.py
index 0cbc93804..911bead15 100644
--- a/django/library/tasks.py
+++ b/django/library/tasks.py
@@ -1,4 +1,15 @@
from huey.contrib.djhuey import db_task, on_commit_task
+from django.conf import settings
+
+from .models import (
+ Codebase,
+ CodebaseGitRemote,
+ CodebaseRelease,
+ GitRefSyncState,
+ ImportedReleaseSyncState,
+)
+from .github_integration import GitHubApi
+from .fs import CodebaseGitRepositoryApi
import logging
@@ -6,12 +17,141 @@
@db_task(retries=1, retry_delay=30)
-def update_fs_release_metadata(release_id: int):
- from .models import CodebaseRelease
+def build_local_git_repo(codebase_id: int):
+ """Build or update the local git mirror repository for a codebase without pushing.
+
+ Safe to call multiple times: will append new releases if mirror exists and has prior build state,
+ otherwise rebuilds from scratch.
+ """
+ codebase = Codebase.objects.get(id=codebase_id)
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ # update_or_build handles both first-time build and incremental append
+ git_fs_api.update_or_build()
+
+
+@db_task(retries=1, retry_delay=30)
+def append_releases_to_local_git_repo(codebase_id: int):
+ """Appends local releases to git mirror."""
+ codebase = Codebase.objects.get(id=codebase_id)
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ git_fs_api.append_releases()
+
+
+@db_task(retries=1, retry_delay=30)
+def import_github_release_task(
+ codebase_id: int,
+ remote_id: int,
+ github_release_id: str,
+ custom_version: str | None = None,
+):
+ """Asynchronously import a GitHub release for a given remote using pre-fetched metadata stored on the sync state.
+
+ The sync state should have been created and populated in the API layer.
+ On failure, marks the state as ERROR.
+ """
+ from .github_integration import GitHubReleaseImporter
+
+ remote = CodebaseGitRemote.objects.get(id=remote_id, codebase__id=codebase_id)
+
+ try:
+ # sync state must exist already
+ sync_state = ImportedReleaseSyncState.objects.filter(
+ remote=remote, github_release_id=str(github_release_id)
+ ).first()
+ if not sync_state:
+ raise ValueError("Missing sync state for GitHub release import")
+ importer = GitHubReleaseImporter(
+ remote=remote, github_release_id=str(github_release_id)
+ )
+ importer.import_or_reimport(custom_version=custom_version)
+ except Exception as e:
+ try:
+ sync_state = ImportedReleaseSyncState.objects.filter(
+ remote=remote, github_release_id=str(github_release_id)
+ ).first()
+ if sync_state:
+ sync_state.log_failure(str(e))
+ except Exception:
+ logger.exception("Failed to update sync state after import error")
+ logger.exception("Failed to import GitHub release asynchronously: %s", e)
+
+
+@db_task(retries=1, retry_delay=30)
+def update_local_repo_release_branch(release_id: int):
+ """asynchronous task that updates a SINGLE RELEASE BRANCH with any metadata changes
+ that may have occurred.
+
+ This should be called when release metadata has been changed
+ """
+ release = CodebaseRelease.objects.get(id=release_id)
+ codebase = release.codebase
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ local_repo = git_fs_api.update_release_branch(release)
+
+
+@db_task(retries=1, retry_delay=30)
+def push_all_releases_to_github(codebase_id: int, remote_id: int):
+ """Manually push all releases to GitHub for a remote, one-by-one."""
+ codebase = Codebase.objects.get(id=codebase_id)
+ remote = CodebaseGitRemote.objects.get(id=remote_id, codebase__id=codebase_id)
+ git_fs_api = CodebaseGitRepositoryApi(codebase)
+ # ensure local repo is up-to-date before pushing
+ local_repo = git_fs_api.update_or_build()
+ gh_api = GitHubApi(codebase=codebase, local_repo=local_repo, remote=remote)
+ # ensure remote exists and persist url
+ repo = gh_api.get_or_create_repo()
+ if remote.url != repo.html_url:
+ remote.url = repo.html_url
+ remote.save(update_fields=["url", "last_modified"])
+
+ # push main first if job started
+ main_state = GitRefSyncState.for_codebase(codebase)
+ if main_state.status == GitRefSyncState.Status.RUNNING:
+ try:
+ commit_sha, summary = gh_api.push_main(local_repo)
+ main_state.record_push_success(commit_sha, summary)
+ except Exception as e:
+ main_state.log_failure(str(e))
+ logger.exception("Failed to push main: %s", e)
+
+ # iterate releases in ascending semver order
+ releases = codebase.ordered_releases_list(internal_only=True, asc=True)
+ for release in releases:
+ state = release.get_or_create_git_ref_sync_state()
+ # skip if job not started
+ if state.status != GitRefSyncState.Status.RUNNING:
+ continue
+ try:
+ commit_sha, summary = gh_api.push_release(
+ local_repo,
+ release,
+ branch_name=state.branch_name,
+ tag_name=state.tag_name,
+ )
+ # create a release for this tag on GitHub if needed
+ gh_api.create_release_for_tag(local_repo, release.version_number)
+ state.record_push_success(commit_sha, summary)
+ except Exception as e:
+ state.log_failure(str(e))
+ # continue with remaining releases, keeping per-release isolation
+ logger.exception("Failed to push release %s: %s", release.version_number, e)
+ continue
+
+
+@db_task(retries=1, retry_delay=30)
+def update_fs_release_metadata(release_id: int):
release = CodebaseRelease.objects.get(id=release_id)
+ codebase = release.codebase
fs_api = release.get_fs_api()
- fs_api.rebuild(metadata_only=True)
+ fs_api.rebuild_metadata()
+ # if the release is published and there is a git ref sync state for this release, update it with
+ # a new commit
+ if (
+ release.is_published
+ and getattr(release, "git_ref_sync_state", None)
+ ):
+ update_local_repo_release_branch(release_id)
@on_commit_task()
diff --git a/django/library/tests/base.py b/django/library/tests/base.py
index 001478b69..7ea8bb20a 100644
--- a/django/library/tests/base.py
+++ b/django/library/tests/base.py
@@ -1,10 +1,11 @@
import io
import logging
+from pathlib import Path
import random
from uuid import UUID
from core.tests.base import ContentModelFactory, UserFactory
-from library.fs import FileCategoryDirectories
+from library.fs import FileCategories
from library.models import (
Codebase,
CodebaseRelease,
@@ -19,6 +20,9 @@
)
from library.serializers import CodebaseSerializer
+TEST_SAMPLES_DIR = Path("library/tests/samples")
+
+
logger = logging.getLogger(__name__)
@@ -179,7 +183,7 @@ class ReleaseSetup:
PROGRAMMING_LANGUAGES = ["Python", "TypeScript"]
@classmethod
- def setUpPublishableDraftRelease(cls, codebase):
+ def setUpPublishableDraftRelease(cls, codebase, with_files=True):
draft_release = codebase.create_release(
status=CodebaseRelease.Status.DRAFT,
initialize=True,
@@ -196,15 +200,14 @@ def setUpPublishableDraftRelease(cls, codebase):
release_contributor_factory = ReleaseContributorFactory(draft_release)
contributor = contributor_factory.create()
release_contributor_factory.create(contributor)
-
- code_file = io.BytesIO(b"print('hello world')")
- code_file.name = "some_code_file.py"
- docs_file = io.BytesIO(b"# Documentation")
- docs_file.name = "some_doc_file.md"
- fs_api = draft_release.get_fs_api()
- fs_api.add(content=code_file, category=FileCategoryDirectories.code)
- fs_api.add(content=docs_file, category=FileCategoryDirectories.docs)
+ if with_files:
+ code_file = io.BytesIO(b"print('hello world')")
+ code_file.name = "some_code_file.py"
+ docs_file = io.BytesIO(b"# Documentation")
+ docs_file.name = "some_doc_file.md"
+ fs_api = draft_release.get_fs_api()
+ fs_api.add(content=code_file, category=FileCategories.code)
+ fs_api.add(content=docs_file, category=FileCategories.docs)
draft_release.save()
-
return draft_release
diff --git a/django/library/tests/archives/.gitignore b/django/library/tests/samples/archives/.gitignore
similarity index 100%
rename from django/library/tests/archives/.gitignore
rename to django/library/tests/samples/archives/.gitignore
diff --git a/django/library/tests/archives/invalid.zip b/django/library/tests/samples/archives/invalid.zip
similarity index 100%
rename from django/library/tests/archives/invalid.zip
rename to django/library/tests/samples/archives/invalid.zip
diff --git a/django/library/tests/archives/nestedcode/.DS_Store b/django/library/tests/samples/archives/nestedcode/.DS_store
similarity index 100%
rename from django/library/tests/archives/nestedcode/.DS_Store
rename to django/library/tests/samples/archives/nestedcode/.DS_store
diff --git a/django/library/tests/archives/nestedcode/.svn/svn_files_here b/django/library/tests/samples/archives/nestedcode/.svn/svn_files_here
similarity index 100%
rename from django/library/tests/archives/nestedcode/.svn/svn_files_here
rename to django/library/tests/samples/archives/nestedcode/.svn/svn_files_here
diff --git a/django/library/tests/archives/nestedcode/README.md b/django/library/tests/samples/archives/nestedcode/README.md
similarity index 100%
rename from django/library/tests/archives/nestedcode/README.md
rename to django/library/tests/samples/archives/nestedcode/README.md
diff --git a/django/library/tests/archives/nestedcode/src/ex.py b/django/library/tests/samples/archives/nestedcode/src/ex.py
similarity index 100%
rename from django/library/tests/archives/nestedcode/src/ex.py
rename to django/library/tests/samples/archives/nestedcode/src/ex.py
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py b/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py
new file mode 100644
index 000000000..8cde7829c
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/code/model.py
@@ -0,0 +1 @@
+print("hello world")
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv b/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv
new file mode 100644
index 000000000..29aa17e68
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/data/input.csv
@@ -0,0 +1,2 @@
+horses,sheep
+10,15
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt b/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt
new file mode 100644
index 000000000..c9be3ff53
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/docs/README.txt
@@ -0,0 +1,3 @@
+instructions:
+
+python model.py
diff --git a/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt b/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt
new file mode 100644
index 000000000..29fee6381
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/1.0.0/results/analysis.txt
@@ -0,0 +1 @@
+result tells us there are more sheep
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py
new file mode 100644
index 000000000..f4963717d
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/cow.py
@@ -0,0 +1,3 @@
+class Cow:
+ def __init__(self, name):
+ self.name = name
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py
new file mode 100644
index 000000000..84790eaf9
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/horse.py
@@ -0,0 +1,3 @@
+class Horse:
+ def __init__(self, name):
+ self.name = name
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py
new file mode 100644
index 000000000..3c575356d
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/animals/sheep.py
@@ -0,0 +1,3 @@
+class Sheep:
+ def __init__(self, name):
+ self.name = name
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py b/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py
new file mode 100644
index 000000000..8cde7829c
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/code/model.py
@@ -0,0 +1 @@
+print("hello world")
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv b/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv
new file mode 100644
index 000000000..603cf0ed3
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/data/input.csv
@@ -0,0 +1,2 @@
+horses,sheep,cows
+10,15,5
diff --git a/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md b/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md
new file mode 100644
index 000000000..64258c3b0
--- /dev/null
+++ b/django/library/tests/samples/releases/animals-model/2.0.0/docs/README.md
@@ -0,0 +1,5 @@
+# instructions:
+
+```
+python model.py
+```
\ No newline at end of file
diff --git a/django/library/tests/test_fs.py b/django/library/tests/test_fs.py
index 5bcd21f9a..b92863816 100644
--- a/django/library/tests/test_fs.py
+++ b/django/library/tests/test_fs.py
@@ -1,32 +1,38 @@
from pathlib import Path
-
+from git import Repo
from django.test import TestCase
+from django.conf import settings
from core.tests.base import (
UserFactory,
destroy_test_shared_folders,
initialize_test_shared_folders,
+ clear_test_shared_folder,
)
from library.fs import (
- FileCategoryDirectories,
+ FileCategories,
StagingDirectories,
MessageLevels,
import_archive,
+ CodebaseGitRepositoryApi,
)
-from library.tests.base import CodebaseFactory
+from library.tests.base import CodebaseFactory, TEST_SAMPLES_DIR
+from library.models import License, GitRefSyncState, ProgrammingLanguage
import logging
logger = logging.getLogger(__name__)
+CODEBASE_CONTENT_CATEGORIES = ["code", "data", "docs"]
+
def setUpModule():
initialize_test_shared_folders()
class ArchiveExtractorTestCase(TestCase):
- nested_code_folder = Path("library/tests/archives/nestedcode")
+ nested_code_folder = TEST_SAMPLES_DIR / "archives" / "nestedcode"
def setUp(self):
self.user_factory = UserFactory()
@@ -47,44 +53,200 @@ def test_zipfile_saving(self):
self.assertEqual(len(logs), 2)
self.assertEqual(
set(
- fs_api.list(StagingDirectories.originals, FileCategoryDirectories.code)
+ fs_api.list(StagingDirectories.originals, FileCategories.code)
),
{"nestedcode.zip"},
)
# Notice that .DS_Store and .svn folder file are eliminated
self.assertEqual(
- set(fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)),
+ set(fs_api.list(StagingDirectories.sip, FileCategories.code)),
{"src/ex.py", "README.md"},
)
fs_api.get_or_create_sip_bag(self.codebase_release.bagit_info)
- fs_api.clear_category(FileCategoryDirectories.code)
+ fs_api.clear_category(FileCategories.code)
self.assertEqual(
set(
- fs_api.list(StagingDirectories.originals, FileCategoryDirectories.code)
+ fs_api.list(StagingDirectories.originals, FileCategories.code)
),
set(),
)
self.assertEqual(
- set(fs_api.list(StagingDirectories.sip, FileCategoryDirectories.code)),
+ set(fs_api.list(StagingDirectories.sip, FileCategories.code)),
set(),
)
def test_invalid_zipfile_saving(self):
- archive_name = "library/tests/archives/invalid.zip"
+ archive_name = str(TEST_SAMPLES_DIR / "archives" / "invalid.zip")
fs_api = self.codebase_release.get_fs_api()
with open(archive_name, "rb") as f:
msgs = fs_api.add(
- FileCategoryDirectories.code, content=f, name="invalid.zip"
+ FileCategories.code, content=f, name="invalid.zip"
)
logs, level = msgs.serialize()
self.assertEqual(level, MessageLevels.error)
self.assertEqual(len(logs), 1)
+ def tearDown(self):
+ clear_test_shared_folder(settings.REPOSITORY_ROOT)
+
@classmethod
def tearDownClass(cls):
super().tearDownClass()
cls.nested_code_folder.with_suffix(".zip").unlink(missing_ok=True)
+class GitRepoApiTestCase(TestCase):
+ model_dir = TEST_SAMPLES_DIR / "releases" / "animals-model"
+ release_1_dir = model_dir / "1.0.0"
+ release_2_dir = model_dir / "2.0.0"
+
+ def setUp(self):
+ self.user_factory = UserFactory()
+ self.submitter = self.user_factory.create()
+ self.codebase_factory = CodebaseFactory(submitter=self.submitter)
+ self.codebase = self.codebase_factory.create()
+ self.release_1 = self.codebase.create_release()
+
+ def tearDown(self):
+ clear_test_shared_folder(settings.REPOSITORY_ROOT)
+
+ def _verify_git_repo_status(self, api, release_count):
+ """Verify git repo structure and contents are correct."""
+ repo = Repo(api.repo_dir)
+ self.assertFalse(repo.is_dirty())
+ self.assertEqual(sum(1 for _ in repo.iter_commits()), release_count)
+ self.assertEqual(len(repo.tags), release_count)
+ # check contents
+ self.assertTrue((api.repo_dir / "codemeta.json").exists())
+ self.assertTrue((api.repo_dir / "CITATION.cff").exists())
+ self.assertTrue((api.repo_dir / "LICENSE").exists())
+
+ def test_repo_build(self):
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_1.publish()
+ public_release_count = self.codebase.releases.public().count()
+ self.assertEqual(public_release_count, 1)
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ # check that the git ref sync states are created and the repo is built
+ built_releases_count = GitRefSyncState.objects.filter(
+ release__codebase=self.codebase, built_commit_sha__isnull=False
+ ).count()
+ self.assertEqual(built_releases_count, 1)
+ self.assertTrue(api.repo_dir.exists())
+ self._verify_git_repo_status(api, public_release_count)
+ fs_api = self.release_1.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ self.assertTrue(
+ api.dirs_equal(
+ fs_api.sip_contents_dir / category,
+ api.repo_dir / category,
+ )
+ )
+
+ def test_repo_append_releases(self):
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_1.publish()
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ self.release_2 = self.codebase.create_release()
+ update_release_from_sample(
+ self.release_2, self.release_2_dir, version_number="2.0.0"
+ )
+ self.release_2.publish()
+ api.append_releases()
+
+ built_releases_count = GitRefSyncState.objects.filter(
+ release__codebase=self.codebase, built_commit_sha__isnull=False
+ ).count()
+ self.assertEqual(built_releases_count, 2)
+ public_release_count = self.codebase.releases.public().count()
+ self._verify_git_repo_status(api, public_release_count)
+ fs_api = self.release_2.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ self.assertTrue(
+ api.dirs_equal(
+ fs_api.sip_contents_dir / category,
+ api.repo_dir / category,
+ )
+ )
+
+ def test_will_not_append_lower_version(self):
+ # publish 2.0.0 first and build
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_2 = self.codebase.create_release()
+ update_release_from_sample(
+ self.release_2, self.release_2_dir, version_number="2.0.0"
+ )
+ self.release_2.publish()
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ # now publish release 1.0.0
+ self.release_1.publish()
+ self.assertRaises(ValueError, api.append_releases)
+
+ def test_repo_rebuild(self):
+ update_release_from_sample(
+ self.release_1, self.release_1_dir, version_number="1.0.0"
+ )
+ self.release_1.publish()
+ api = CodebaseGitRepositoryApi(self.codebase)
+ api.build()
+ self.release_2 = self.codebase.create_release()
+ update_release_from_sample(
+ self.release_2, self.release_2_dir, version_number="2.0.0"
+ )
+ self.release_2.publish()
+ api.build()
+
+ built_releases_count = GitRefSyncState.objects.filter(
+ release__codebase=self.codebase, built_commit_sha__isnull=False
+ ).count()
+ self.assertEqual(built_releases_count, 2)
+ public_release_count = self.codebase.releases.public().count()
+ self._verify_git_repo_status(api, public_release_count)
+ fs_api = self.release_2.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ self.assertTrue(
+ api.dirs_equal(
+ fs_api.sip_contents_dir / category,
+ api.repo_dir / category,
+ )
+ )
+
+
def tearDownModule():
destroy_test_shared_folders()
+
+
+# helpers ================================================
+
+
+def upload_category(fs_api, release_dir: Path, category: str):
+ category_path = release_dir / category
+ for filepath in category_path.rglob("*"):
+ if filepath.is_file():
+ with filepath.open("rb") as f:
+ relpath = filepath.relative_to(category_path)
+ file_name = str(relpath)
+ fs_api.add(FileCategories[category], content=f, name=file_name)
+
+
+def update_release_from_sample(release, sample_dir, version_number):
+ release.os = "Linux"
+ python, _created = ProgrammingLanguage.objects.get_or_create_by_name("Python")
+ release.programming_languages.add(python)
+ release.license = License.objects.create(name="MIT")
+ release.release_notes = "Initial release"
+ release.version_number = version_number
+ release.save()
+ fs_api = release.get_fs_api()
+ for category in CODEBASE_CONTENT_CATEGORIES:
+ upload_category(fs_api, sample_dir, category)
+ return release
diff --git a/django/library/tests/test_github_integration.py b/django/library/tests/test_github_integration.py
new file mode 100644
index 000000000..1092f4cad
--- /dev/null
+++ b/django/library/tests/test_github_integration.py
@@ -0,0 +1,377 @@
+from unittest.mock import patch, MagicMock
+from django.test import TestCase
+from github.GithubException import UnknownObjectException
+
+from core.tests.base import UserFactory
+from library.github_integration import (
+ GitHubApi,
+ GitHubRepoValidator,
+ GitHubReleaseImporter,
+ _serialize_github_release_listing_item,
+ extract_semver,
+)
+from library.models import (
+ CodebaseGitRemote,
+ GithubIntegrationAppInstallation,
+ CodebaseRelease,
+ ImportedReleaseSyncState,
+)
+from library.tests.base import CodebaseFactory
+
+SAMPLE_PAYLOAD = {
+ "action": "released",
+ "release": {
+ "id": 12345,
+ "tag_name": "v1.0.0",
+ "name": "Version 1.0.0",
+ "body": "Initial release",
+ "draft": False,
+ "prerelease": False,
+ "zipball_url": "https://api.github.com/repos/testuser/test-repo/zipball/v1.0.0",
+ "html_url": "https://github.com/testuser/test-repo/releases/tag/v1.0.0",
+ },
+ "repository": {
+ "name": "test-repo",
+ "owner": {"login": "testuser"},
+ "private": False,
+ },
+ "installation": {"id": 54321},
+}
+
+
+class GitHubRepoValidatorTests(TestCase):
+ def setUp(self):
+ # set up a user and installation for testing validator methods
+ self.user = UserFactory().create()
+ self.installation = GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ github_login="testuser",
+ installation_id=1,
+ github_user_id=123,
+ )
+
+ def test_validate_format(self):
+ # test valid repo names
+ for name in ["repo", "repo-1", "repo.1", "my_repo"]:
+ with self.subTest(name=name):
+ validator = GitHubRepoValidator(name)
+ self.assertIsNone(validator.validate_format())
+
+ # test invalid repo names
+ for name in [
+ "repo with space",
+ "repo!",
+ "a" * 101,
+ "repo.git",
+ "contains-github",
+ ]:
+ with self.subTest(name=name):
+ validator = GitHubRepoValidator(name)
+ with self.assertRaises(ValueError):
+ validator.validate_format()
+
+
+class GitHubReleaseImporterTests(TestCase):
+ def setUp(self):
+ # set up a user, codebase, and remote for importing releases
+ self.user = UserFactory().create()
+ self.installation = GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ github_login="testuser",
+ installation_id=54321,
+ github_user_id=123,
+ )
+ self.codebase = CodebaseFactory(submitter=self.user).create()
+ self.remote = CodebaseGitRemote.objects.create(
+ codebase=self.codebase,
+ owner="testuser",
+ repo_name="test-repo",
+ is_user_repo=True,
+ is_active=True,
+ )
+ self.payload = SAMPLE_PAYLOAD.copy()
+ # create sync state from payload
+ self.sync_state = ImportedReleaseSyncState.for_github_release(
+ self.remote, self.payload["release"]
+ )
+
+ def create_importer(self, github_release_id="12345"):
+ return GitHubReleaseImporter(self.remote, github_release_id)
+
+ def test_init_success(self):
+ # should initialize successfully with a valid remote and release id
+ importer = self.create_importer("12345")
+ self.assertEqual(importer.github_release_id, "12345")
+ self.assertEqual(importer.codebase, self.codebase)
+
+ def test_init_failures(self):
+ # test various invalid scenarios that should raise ValueError
+ # missing sync state
+ with self.assertRaises(ValueError):
+ self.create_importer("99999")
+
+ # sync state without download_url
+ ImportedReleaseSyncState.objects.create(
+ remote=self.remote,
+ github_release_id="88888",
+ download_url="", # empty download url
+ )
+ with self.assertRaises(ValueError):
+ self.create_importer("88888")
+
+ def test_extract_semver(self):
+ # delegate test, see ExtractSemverTests for exhaustive coverage
+ importer = self.create_importer("12345")
+ self.assertEqual(importer.extract_semver("v1.2.3"), "1.2.3")
+
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_import_new_release(self, mock_get_token, mock_get_fs_api, mock_get_repo):
+ # mock token and fs_api calls
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {"name": "test-repo", "full_name": "testuser/test-repo"}
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {}) # codemeta, cff
+ mock_get_fs_api.return_value = mock_fs_api
+
+ # import a new release
+ importer = self.create_importer("12345")
+ success = importer.import_new_release()
+
+ # check that it was successful and objects were created
+ self.assertTrue(success)
+ self.assertTrue(
+ CodebaseRelease.objects.filter(
+ codebase=self.codebase, version_number="1.0.0"
+ ).exists()
+ )
+ release = CodebaseRelease.objects.get(version_number="1.0.0")
+ self.assertEqual(release.imported_release_sync_state.github_release_id, "12345")
+ self.assertEqual(release.submitter, self.codebase.submitter)
+ mock_fs_api.import_release_package.assert_called_once()
+
+ @patch("library.github_integration.GitHubApi.get_release_raw_for_remote")
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_reimport_release(self, mock_get_token, mock_get_fs_api, mock_get_repo, mock_get_release):
+ # mock token and fs_api calls
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {"name": "test-repo", "full_name": "testuser/test-repo"}
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {}) # codemeta, cff
+ mock_get_fs_api.return_value = mock_fs_api
+
+ # first, import a new release
+ importer = self.create_importer("12345")
+ importer.import_or_reimport()
+
+ self.assertEqual(CodebaseRelease.objects.count(), 1)
+ release = CodebaseRelease.objects.first()
+ self.assertEqual(
+ release.imported_release_sync_state.download_url,
+ "https://api.github.com/repos/testuser/test-repo/zipball/v1.0.0",
+ )
+
+ # now, create a new importer with an updated payload for an "edited" event
+ reimport_payload = self.payload.copy()
+ reimport_payload["action"] = "edited"
+ reimport_payload["release"] = reimport_payload["release"].copy()
+ new_url = (
+ "https://api.github.com/repos/testuser/test-repo/zipball/v1.0.0-updated"
+ )
+ reimport_payload["release"]["zipball_url"] = new_url
+ # mock get_release_raw_for_remote to return the updated payload
+ mock_get_release.return_value = reimport_payload["release"]
+
+ importer2 = self.create_importer("12345")
+ success = importer2.import_or_reimport()
+
+ # assert that the re-import was successful and the release was updated
+ self.assertTrue(success)
+ self.assertEqual(mock_fs_api.import_release_package.call_count, 2)
+
+ release.refresh_from_db()
+ self.assertEqual(release.imported_release_sync_state.download_url, new_url)
+ # release version number should NOT have changed
+ self.assertEqual(release.version_number, "1.0.0")
+
+ @patch("library.github_integration.GitHubApi.get_release_raw_for_remote")
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_reimport_skips_when_download_url_unchanged(
+ self, mock_get_token, mock_get_fs_api, mock_get_repo, mock_get_release
+ ):
+ # when the zipball URL has not changed, reimport_release should detect no
+ # changes and return without calling import_release_package
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {"name": "test-repo", "full_name": "testuser/test-repo"}
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {})
+ mock_get_fs_api.return_value = mock_fs_api
+
+ # perform an initial import so a linked release exists
+ importer = self.create_importer("12345")
+ first_success = importer.import_or_reimport()
+ self.assertTrue(first_success)
+ self.assertEqual(mock_fs_api.import_release_package.call_count, 1)
+
+ # mock get_release to return the same zipball_url as the cached sync state
+ original_url = self.sync_state.download_url
+ mock_get_release.return_value = {
+ "id": 12345,
+ "tag_name": "v1.0.0",
+ "name": "Version 1.0.0",
+ "zipball_url": original_url,
+ "html_url": "https://github.com/testuser/test-repo/releases/tag/v1.0.0",
+ }
+
+ importer2 = self.create_importer("12345")
+ success = importer2.import_or_reimport()
+
+ # import_release_package must NOT be called a second time
+ self.assertTrue(success)
+ self.assertEqual(mock_fs_api.import_release_package.call_count, 1)
+
+ @patch("library.github_integration.GitHubApi.get_repo_raw_for_remote")
+ @patch("library.models.CodebaseRelease.get_fs_api")
+ @patch("library.github_integration.GitHubApi.get_user_installation_access_token")
+ def test_import_new_release_version_collision(
+ self, mock_get_token, mock_get_fs_api, mock_get_repo
+ ):
+ # even with valid metadata/api mocks, import should fail when version already exists
+ mock_get_token.return_value = "fake-token"
+ mock_get_repo.return_value = {
+ "name": "test-repo",
+ "full_name": "testuser/test-repo",
+ }
+ mock_fs_api = MagicMock()
+ mock_fs_api.import_release_package.return_value = ({}, {})
+ mock_get_fs_api.return_value = mock_fs_api
+
+ existing_release = self.codebase.create_release()
+ existing_release.version_number = "1.0.0"
+ existing_release.save()
+
+ importer = self.create_importer("12345")
+ success = importer.import_new_release()
+
+ self.assertFalse(success)
+ self.assertEqual(
+ CodebaseRelease.objects.filter(codebase=self.codebase, version_number="1.0.0").count(),
+ 1,
+ )
+ mock_fs_api.import_release_package.assert_not_called()
+
+
+class ExtractSemverTests(TestCase):
+ def test_standard_version(self):
+ self.assertEqual(extract_semver("1.2.3"), "1.2.3")
+
+ def test_leading_v(self):
+ self.assertEqual(extract_semver("v1.2.3"), "1.2.3")
+
+ def test_embedded_in_string(self):
+ self.assertEqual(extract_semver("version 1.2.3-beta"), "1.2.3")
+
+ def test_two_part_version_returns_none(self):
+ self.assertIsNone(extract_semver("1.2"))
+
+ def test_non_version_string_returns_none(self):
+ self.assertIsNone(extract_semver("invalid-version"))
+
+ def test_empty_string_returns_none(self):
+ self.assertIsNone(extract_semver(""))
+
+ def test_non_string_returns_none(self):
+ self.assertIsNone(extract_semver(None))
+ self.assertIsNone(extract_semver(123))
+
+ def test_oversized_input_returns_none(self):
+ self.assertIsNone(extract_semver("1.2.3" * 300))
+
+
+class GitHubIntegrationHelpersTests(TestCase):
+ def test_serialize_github_release_listing_item_prefers_raw_data(self):
+ release = MagicMock()
+ release.id = 999
+ release.raw_data = {
+ "id": 12345,
+ "tag_name": "v1.2.3",
+ "name": "Release 1.2.3",
+ "html_url": "https://github.com/acme/repo/releases/tag/v1.2.3",
+ "zipball_url": "https://api.github.com/repos/acme/repo/zipball/v1.2.3",
+ "draft": False,
+ "prerelease": False,
+ "created_at": "2025-01-01T00:00:00Z",
+ "published_at": "2025-01-02T00:00:00Z",
+ }
+
+ data, raw = _serialize_github_release_listing_item(release)
+
+ self.assertEqual(raw["id"], 12345)
+ self.assertEqual(data["id"], "12345")
+ self.assertEqual(data["tag_name"], "v1.2.3")
+ self.assertEqual(data["version"], "1.2.3")
+ self.assertTrue(data["has_semantic_versioning"])
+
+ def test_serialize_github_release_listing_item_falls_back_to_attributes(self):
+ release = MagicMock()
+ release.raw_data = {}
+ release.id = 555
+ release.tag_name = "2.0.1"
+ release.title = None
+ release.name = "Project Release 2.0.1"
+ release.html_url = "https://github.com/acme/repo/releases/tag/2.0.1"
+ release.zipball_url = "https://api.github.com/repos/acme/repo/zipball/2.0.1"
+ release.draft = True
+ release.prerelease = False
+ release.created_at = "2025-03-01T00:00:00Z"
+ release.published_at = "2025-03-02T00:00:00Z"
+
+ data, _ = _serialize_github_release_listing_item(release)
+
+ self.assertEqual(data["id"], "555")
+ self.assertEqual(data["name"], "Project Release 2.0.1")
+ self.assertEqual(data["version"], "2.0.1")
+ self.assertTrue(data["has_semantic_versioning"])
+ self.assertTrue(data["draft"])
+
+ def test_create_release_for_tag_skips_when_release_exists(self):
+ api = GitHubApi.__new__(GitHubApi)
+ api._github_repo = MagicMock()
+ local_repo = MagicMock()
+
+ api._github_repo.get_release.return_value = MagicMock()
+
+ api.create_release_for_tag(local_repo, "v1.0.0")
+
+ api._github_repo.get_release.assert_called_once_with("v1.0.0")
+ api._github_repo.create_git_release.assert_not_called()
+
+ def test_create_release_for_tag_creates_with_tag_message(self):
+ api = GitHubApi.__new__(GitHubApi)
+ api._github_repo = MagicMock()
+ local_repo = MagicMock()
+
+ api._github_repo.get_release.side_effect = UnknownObjectException(
+ 404, {"message": "Not Found"}, None
+ )
+ tag = MagicMock()
+ tag.name = "v1.0.0"
+ tag.commit.message = "Release notes"
+ local_repo.tags = [tag]
+
+ api.create_release_for_tag(local_repo, "v1.0.0")
+
+ api._github_repo.create_git_release.assert_called_once_with(
+ "v1.0.0",
+ name="v1.0.0",
+ message="Release notes",
+ draft=False,
+ prerelease=False,
+ )
+
+
diff --git a/django/library/tests/test_metadata.py b/django/library/tests/test_metadata.py
index dadf08e05..d94155804 100644
--- a/django/library/tests/test_metadata.py
+++ b/django/library/tests/test_metadata.py
@@ -1,11 +1,13 @@
import logging
+from codemeticulous.cff.models import LicenseEnum
+from django.test import TestCase
from core.tests.base import BaseModelTestCase, UserFactory
from .base import (
CodebaseFactory,
ReleaseSetup,
)
-from library.metadata import CodeMeta
+from library.metadata import CodeMeta, ReleaseMetadataConverter
logger = logging.getLogger(__name__)
@@ -90,4 +92,199 @@ def recurse_contents(contents):
recurse_contents(item["contents"])
recurse_contents(sip_contents["contents"])
- self.assertEqual(len(files_found), 3)
+ self.assertEqual(
+ set(label.lower() for label in files_found),
+ {"license", "citation.cff", "codemeta.json"},
+ )
+
+
+class ReleaseMetadataConverterTestCase(TestCase):
+
+ def setUp(self):
+ self.minimal_codemeta = {"name": "Test Code"}
+ self.minimal_cff = {
+ "cff-version": "1.2.0",
+ "title": "Test Title",
+ "message": "Test Message",
+ "authors": [{"name": "Test Author"}],
+ }
+
+ def test_extract_license_from_github(self):
+ converter = ReleaseMetadataConverter(
+ github_repository={"license": {"spdx_id": "MIT"}}
+ )
+ self.assertEqual(converter._extract_license_from_github(), "MIT")
+
+ def test_extract_license_from_cff(self):
+ cff = {**self.minimal_cff, "license": LicenseEnum.GPL_3_0_only}
+ converter = ReleaseMetadataConverter(cff=cff)
+ self.assertEqual(converter._extract_license_from_cff(), "GPL-3.0-only")
+
+ def test_extract_release_notes_from_codemeta(self):
+ codemeta = {**self.minimal_codemeta, "releaseNotes": "From CodeMeta"}
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(
+ converter._extract_release_notes_from_codemeta(), "From CodeMeta"
+ )
+
+ def test_extract_release_notes_from_github(self):
+ converter = ReleaseMetadataConverter(github_release={"body": "From GitHub"})
+ self.assertEqual(converter._extract_release_notes_from_github(), "From GitHub")
+
+ def test_extract_os_from_codemeta(self):
+ codemeta = {**self.minimal_codemeta, "operatingSystem": "Linux"}
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(converter._extract_os_from_codemeta(), "linux")
+
+ def test_extract_os_from_codemeta_variants(self):
+ cases = [
+ ("macOS Sonoma", "macos"),
+ ("Windows 11", "windows"),
+ ("platform independent", "platform_independent"),
+ ("Plan9", "other"),
+ ]
+ for os_value, expected in cases:
+ with self.subTest(os_value=os_value):
+ converter = ReleaseMetadataConverter(
+ codemeta={**self.minimal_codemeta, "operatingSystem": os_value}
+ )
+ self.assertEqual(converter._extract_os_from_codemeta(), expected)
+
+ def test_extract_os_from_codemeta_structured_object(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "operatingSystem": {"name": "Any"},
+ "releaseNotes": "From CodeMeta",
+ }
+ converter = ReleaseMetadataConverter(
+ codemeta=codemeta,
+ github_repository={"license": {"spdx_id": "MIT"}, "language": "Python"},
+ github_release={"body": "From GitHub"},
+ )
+ # Unsupported structured OS input should be ignored without breaking conversion.
+ self.assertEqual(converter._extract_os_from_codemeta(), "")
+ result = converter.convert()
+ self.assertEqual(result["os"], "")
+ self.assertEqual(result["release_notes"], "From GitHub")
+ self.assertEqual(result["license_spdx_id"], "MIT")
+ self.assertEqual(result["programming_languages"], ["Python"])
+
+ def test_extract_programming_languages_from_codemeta(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "programmingLanguage": ["Python", "R"],
+ }
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(
+ converter._extract_programming_languages_from_codemeta(), ["Python", "R"]
+ )
+
+ def test_extract_programming_languages_from_codemeta_edge_cases(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "programmingLanguage": [{"name": "Julia"}, {"foo": "bar"}, 123],
+ }
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ languages = converter._extract_programming_languages_from_codemeta()
+ self.assertEqual(languages, ["Julia", "123"])
+ checked_languages = languages or []
+ self.assertNotIn("bar", checked_languages)
+
+ def test_extract_programming_languages_from_github(self):
+ converter = ReleaseMetadataConverter(github_repository={"language": "Java"})
+ self.assertEqual(
+ converter._extract_programming_languages_from_github(), ["Java"]
+ )
+
+ def test_extract_platforms_from_codemeta(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "runtimePlatform": ["Mesa", "NetLogo"],
+ }
+ converter = ReleaseMetadataConverter(codemeta=codemeta)
+ self.assertEqual(
+ converter._extract_platforms_from_codemeta(), ["Mesa", "NetLogo"]
+ )
+
+ def test_extract_platforms_from_codemeta_edge_cases(self):
+ codemeta = {
+ **self.minimal_codemeta,
+ "runtimePlatform": ["Mesa", {"name": "NetLogo"}, None],
+ "releaseNotes": "From CodeMeta",
+ }
+ converter = ReleaseMetadataConverter(
+ codemeta=codemeta,
+ github_repository={"license": {"spdx_id": "MIT"}, "language": "Python"},
+ github_release={"body": "From GitHub"},
+ )
+ # Mixed structured runtimePlatform input is not supported, but conversion should remain stable.
+ self.assertIsNone(converter._extract_platforms_from_codemeta())
+ result = converter.convert()
+ self.assertIsNone(result["platforms"])
+ self.assertEqual(result["release_notes"], "From GitHub")
+ self.assertEqual(result["license_spdx_id"], "MIT")
+ self.assertEqual(result["programming_languages"], ["Python"])
+
+ def test_extract_release_notes_from_codemeta_list_edge_cases(self):
+ converter = ReleaseMetadataConverter(
+ codemeta={**self.minimal_codemeta, "releaseNotes": ["First note", "Second"]}
+ )
+ self.assertEqual(converter._extract_release_notes_from_codemeta(), "First note")
+
+ converter = ReleaseMetadataConverter(
+ codemeta={**self.minimal_codemeta, "releaseNotes": [{"text": "bad"}]}
+ )
+ self.assertIsNone(converter._extract_release_notes_from_codemeta())
+
+ def test_convert_priority(self):
+ # test the convert() method's priority logic
+ codemeta = {
+ **self.minimal_codemeta,
+ "releaseNotes": "From CodeMeta",
+ "programmingLanguage": ["Python"],
+ }
+ github_release = {"body": "From GitHub"}
+ github_repository = {"license": {"spdx_id": "MIT"}, "language": "Java"}
+ cff = {**self.minimal_cff, "license": LicenseEnum.GPL_3_0_only}
+
+ converter = ReleaseMetadataConverter(
+ codemeta=codemeta,
+ cff=cff,
+ github_repository=github_repository,
+ github_release=github_release,
+ )
+ result = converter.convert()
+ self.assertEqual(result["license_spdx_id"], "MIT") # Github > CFF
+ self.assertEqual(result["release_notes"], "From CodeMeta") # CodeMeta > Github
+ self.assertEqual(
+ result["programming_languages"], ["Java"]
+ ) # Only pull languages from GitHub
+
+ def test_convert_fallback(self):
+ """Test the convert() method's fallback logic."""
+ github_release = {"body": "From GitHub"}
+ github_repository = {"language": "Java"}
+ cff = {**self.minimal_cff, "license": LicenseEnum.GPL_3_0_only}
+
+ converter = ReleaseMetadataConverter(
+ cff=cff,
+ github_repository=github_repository,
+ github_release=github_release,
+ )
+ result = converter.convert()
+ self.assertEqual(result["license_spdx_id"], "GPL-3.0-only")
+ self.assertEqual(result["release_notes"], "From GitHub")
+ self.assertEqual(result["programming_languages"], ["Java"])
+
+ def test_convert_no_sources(self):
+ """Test that convert returns gracefully with no sources."""
+ converter = ReleaseMetadataConverter()
+ result = converter.convert()
+ expected = {
+ "license_spdx_id": None,
+ "release_notes": None,
+ "os": "",
+ "programming_languages": None,
+ "platforms": None,
+ }
+ self.assertEqual(result, expected)
diff --git a/django/library/tests/test_views.py b/django/library/tests/test_views.py
index cf5ac1f54..749568aa6 100644
--- a/django/library/tests/test_views.py
+++ b/django/library/tests/test_views.py
@@ -18,7 +18,6 @@
ApiAccountMixin,
)
from library.forms import PeerReviewerFeedbackReviewerForm
-from library.fs import FileCategoryDirectories
from library.models import (
ProgrammingLanguage,
ReleaseLanguage,
@@ -27,6 +26,7 @@
License,
PeerReview,
)
+from library.fs import FileCategories
from library.tests.base import ReviewSetup
from .base import (
CodebaseFactory,
@@ -35,7 +35,18 @@
PeerReviewInvitationFactory,
ReleaseSetup,
)
-from ..views import CodebaseViewSet, CodebaseReleaseViewSet, PeerReviewInvitationViewSet
+from ..views import (
+ CodebaseViewSet,
+ CodebaseReleaseViewSet,
+ PeerReviewInvitationViewSet,
+ github_sync_webhook,
+)
+from allauth.socialaccount.models import SocialAccount
+import json
+import hmac
+import hashlib
+from django.test import override_settings
+from library.models import GithubIntegrationAppInstallation
import logging
@@ -331,7 +342,7 @@ def test_upload_file(self):
# Unpublished codebase release permissions
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code)
+ api.get_originals_list_url(category=FileCategories.code)
)
self.assertResponseNotFound(response)
for user, expected_status_code in [
@@ -341,7 +352,7 @@ def test_upload_file(self):
]:
self.login(user, self.user_factory.password)
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code),
+ api.get_originals_list_url(category=FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -356,7 +367,7 @@ def test_upload_file(self):
# Published codebase release permissions
self.client.logout()
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code)
+ api.get_originals_list_url(category=FileCategories.code)
)
self.assertResponsePermissionDenied(response)
for user, expected_status_code in [
@@ -366,7 +377,7 @@ def test_upload_file(self):
]:
self.login(user, self.user_factory.password)
response = self.client.post(
- api.get_originals_list_url(category=FileCategoryDirectories.code),
+ api.get_originals_list_url(category=FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -380,7 +391,7 @@ def test_list_files(self):
# Unpublished codebase release permissions
response = self.client.get(
- api.get_originals_list_url(category=FileCategoryDirectories.code)
+ api.get_originals_list_url(category=FileCategories.code)
)
self.assertResponseNotFound(response)
for user, expected_status_code in [
@@ -390,7 +401,7 @@ def test_list_files(self):
]:
self.login(user, self.user_factory.password)
response = self.client.get(
- api.get_originals_list_url(FileCategoryDirectories.code),
+ api.get_originals_list_url(FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -405,7 +416,7 @@ def test_list_files(self):
# Published codebase release permissions
response = self.client.get(
- api.get_originals_list_url(FileCategoryDirectories.code)
+ api.get_originals_list_url(FileCategories.code)
)
self.assertResponsePermissionDenied(response)
for user, expected_status_code in [
@@ -415,7 +426,7 @@ def test_list_files(self):
]:
self.login(user, self.user_factory.password)
response = self.client.get(
- api.get_originals_list_url(FileCategoryDirectories.code),
+ api.get_originals_list_url(FileCategories.code),
HTTP_ACCEPT="application/json",
)
self.assertEqual(
@@ -430,7 +441,7 @@ def test_delete_file(self):
# Unpublished codebase release permissions
response = self.client.delete(
api.get_absolute_url(
- category=FileCategoryDirectories.code, relpath=path_to_foo
+ category=FileCategories.code, relpath=path_to_foo
)
)
self.assertResponseNotFound(response)
@@ -442,7 +453,7 @@ def test_delete_file(self):
self.login(user, self.user_factory.password)
response = self.client.delete(
api.get_absolute_url(
- category=FileCategoryDirectories.code, relpath=path_to_foo
+ category=FileCategories.code, relpath=path_to_foo
),
HTTP_ACCEPT="application/json",
)
@@ -455,7 +466,7 @@ def test_delete_file(self):
# Published codebase release permissions
response = self.client.delete(
api.get_absolute_url(
- category=FileCategoryDirectories.code, relpath=path_to_foo
+ category=FileCategories.code, relpath=path_to_foo
)
)
self.assertResponsePermissionDenied(response)
@@ -466,7 +477,7 @@ def test_delete_file(self):
]:
self.login(user, self.user_factory.password)
response = self.client.delete(
- api.get_absolute_url(FileCategoryDirectories.code, path_to_foo),
+ api.get_absolute_url(FileCategories.code, path_to_foo),
HTTP_ACCEPT="application/json",
)
self.assertEqual(response.status_code, expected_status_code, msg=repr(user))
@@ -551,8 +562,8 @@ def test_publish_codebaserelease(self):
docs_file.name = "README.md"
api = self.codebase_release.get_fs_api()
- api.add(content=code_file, category=FileCategoryDirectories.code)
- api.add(content=docs_file, category=FileCategoryDirectories.docs)
+ api.add(content=code_file, category=FileCategories.code)
+ api.add(content=docs_file, category=FileCategories.docs)
with self.assertRaises(
ValidationError, msg="Codebase has no metadata, should fail publish"
@@ -596,25 +607,27 @@ def test_publish_codebaserelease(self):
class CodebaseRenderPageTestCase(TestCase):
def setUp(self):
- user_factory = UserFactory()
- self.submitter = user_factory.create()
+ self.user_factory = UserFactory()
+ self.submitter = self.user_factory.create()
codebase_factory = CodebaseFactory(submitter=self.submitter)
self.codebase = codebase_factory.create()
+ self.codebase.create_release(
+ status=CodebaseRelease.Status.PUBLISHED, initialize=False
+ )
def test_list(self):
response = self.client.get(
reverse(
- "library:codebase-detail",
- kwargs={"identifier": self.codebase.identifier},
+ "library:codebase-list"
)
)
- self.assertTrue(response.status_code, 200)
+ self.assertEqual(response.status_code, 200)
class CodebaseReleaseRenderPageTestCase(TestCase):
def setUp(self):
- user_factory = UserFactory()
- self.submitter = user_factory.create()
+ self.user_factory = UserFactory()
+ self.submitter = self.user_factory.create()
codebase_factory = CodebaseFactory(submitter=self.submitter)
contributor_factory = ContributorFactory(user=self.submitter)
contributor = contributor_factory.create(user=self.submitter)
@@ -627,6 +640,9 @@ def setUp(self):
release_contributor_factory.create(contributor=contributor)
def test_detail(self):
+ self.client.login(
+ username=self.submitter.username, password=self.user_factory.password
+ )
response = self.client.get(
reverse(
"library:codebaserelease-detail",
@@ -636,7 +652,7 @@ def test_detail(self):
},
)
)
- self.assertTrue(response.status_code, True)
+ self.assertEqual(response.status_code, 200)
class CodebaseSearchTestCase(TestCase):
@@ -755,5 +771,177 @@ def test_cannot_update_feedback_on_complete_review(self):
self.assertFalse(form.is_valid())
+class GitHubWebhookTestCase(ResponseStatusCodesMixin, TestCase):
+ def setUp(self):
+ self.user_factory = UserFactory()
+ self.user = self.user_factory.create()
+ self.path = reverse("library:github-sync-webhook")
+ self.secret = "test-secret"
+ self.factory = RequestFactory()
+
+ def _create_signed_request(self, payload, signature=None):
+ body = json.dumps(payload).encode("utf-8")
+ return self._create_signed_raw_request(body, signature=signature)
+
+ def _create_signed_raw_request(self, body, signature=None, event="installation"):
+ if signature is None:
+ hash_object = hmac.new(
+ self.secret.encode("utf-8"), msg=body, digestmod=hashlib.sha256
+ )
+ signature = f"sha256={hash_object.hexdigest()}"
+ request = self.factory.post(
+ self.path,
+ data=body,
+ content_type="application/json",
+ HTTP_X_GITHUB_EVENT=event,
+ HTTP_X_HUB_SIGNATURE_256=signature,
+ )
+ return request
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_no_matching_social_account(self):
+ # if the github account doesn't match a social account user nothing happens
+ payload = {
+ "action": "created",
+ "installation": {"id": 12345},
+ "sender": {"id": 99999, "login": "newuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 404)
+ self.assertEqual(GithubIntegrationAppInstallation.objects.count(), 0)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_creates_installation(self):
+ # if it does, it creates a installation object for them
+ SocialAccount.objects.create(
+ user=self.user,
+ provider="github",
+ uid="54321",
+ )
+ payload = {
+ "action": "created",
+ "installation": {"id": 12345},
+ "sender": {"id": 54321, "login": "testuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(GithubIntegrationAppInstallation.objects.count(), 1)
+ installation = GithubIntegrationAppInstallation.objects.first()
+ self.assertEqual(installation.user, self.user)
+ self.assertEqual(installation.installation_id, 12345)
+ self.assertEqual(installation.github_login, "testuser")
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_updates_existing_installation(self):
+ # re-installing the app on a user acc that already has the link will update the existing installation object
+ SocialAccount.objects.create(
+ user=self.user,
+ provider="github",
+ uid="54321",
+ )
+ GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ installation_id=11111,
+ github_user_id=54321,
+ github_login="testuser",
+ )
+ payload = {
+ "action": "created",
+ "installation": {"id": 22222},
+ "sender": {"id": 54321, "login": "testuser-renamed"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(GithubIntegrationAppInstallation.objects.count(), 1)
+ installation = GithubIntegrationAppInstallation.objects.first()
+ self.assertEqual(installation.installation_id, 22222)
+ self.assertEqual(installation.github_login, "testuser-renamed")
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_bad_signature(self):
+ # requests with a bad signature should be rejected
+ payload = {"action": "created"}
+ request = self._create_signed_request(payload, signature="sha256=bad-sig")
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 403)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_malformed_installation_payload_invalid_json(self):
+ request = self._create_signed_raw_request(b"{not-json")
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 400)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_malformed_installation_payload_missing_sender(self):
+ payload = {"action": "created", "installation": {"id": 12345}}
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 400)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_malformed_installation_payload_missing_installation_id(self):
+ payload = {
+ "action": "created",
+ "installation": {},
+ "sender": {"id": 54321, "login": "testuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 400)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_missing_signature_header_returns_403(self):
+ # requests that omit the signature header entirely must be rejected
+ body = json.dumps({"action": "created"}).encode("utf-8")
+ request = self.factory.post(
+ self.path,
+ data=body,
+ content_type="application/json",
+ HTTP_X_GITHUB_EVENT="installation",
+ # deliberately no HTTP_X_HUB_SIGNATURE_256
+ )
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 403)
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_installation_webhook_deletes_installation(self):
+ # action=deleted removes the installation record matched by user (via social account uid),
+ # NOT by installation_id — the id in the payload is parsed but unused by the delete path.
+ SocialAccount.objects.create(
+ user=self.user,
+ provider="github",
+ uid="54321",
+ )
+ GithubIntegrationAppInstallation.objects.create(
+ user=self.user,
+ installation_id=12345,
+ github_user_id=54321,
+ github_login="testuser",
+ )
+ # use a mismatched installation id to confirm deletion is keyed on user, not installation_id
+ payload = {
+ "action": "deleted",
+ "installation": {"id": 99999},
+ "sender": {"id": 54321, "login": "testuser"},
+ }
+ request = self._create_signed_request(payload)
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 200)
+ self.assertFalse(
+ GithubIntegrationAppInstallation.objects.filter(user=self.user).exists()
+ )
+
+ @override_settings(GITHUB_INTEGRATION_APP_WEBHOOK_SECRET="test-secret")
+ def test_non_installation_event_returns_202(self):
+ # unrecognised event types should be acknowledged but ignored
+ body = json.dumps({"action": "opened"}).encode("utf-8")
+ request = self._create_signed_raw_request(body, event="pull_request")
+ response = github_sync_webhook(request)
+ self.assertEqual(response.status_code, 202)
+
+
def tearDownModule():
shutil.rmtree(settings.LIBRARY_ROOT, ignore_errors=True)
diff --git a/django/library/urls.py b/django/library/urls.py
index b4e72a705..e51f58c9e 100644
--- a/django/library/urls.py
+++ b/django/library/urls.py
@@ -14,6 +14,11 @@
router.register(
r"codebases/(?P
+
+
+
diff --git a/frontend/src/components/GitHubIntegrationConfiguration.vue b/frontend/src/components/GitHubIntegrationConfiguration.vue
new file mode 100644
index 000000000..0e3f864cc
--- /dev/null
+++ b/frontend/src/components/GitHubIntegrationConfiguration.vue
@@ -0,0 +1,259 @@
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/GitHubIntegrationOverview.vue b/frontend/src/components/GitHubIntegrationOverview.vue
new file mode 100644
index 000000000..4b67fcb29
--- /dev/null
+++ b/frontend/src/components/GitHubIntegrationOverview.vue
@@ -0,0 +1,95 @@
+
+
+
+
+
+
+
+ Successfully connected to {{ activeRemote.owner }}/{{ activeRemote.repoName }}
+
+ + Manage this connection by importing releases from GitHub or pushing releases to GitHub + below. Return here in the future by pressing + manage in the GitHub panel on the model page. +
+
+
+ + This will deactivate the connection to + {{ activeRemote?.owner }}/{{ activeRemote?.repoName }}. +
++ If you reconnect to the same repository later, your push history will be preserved. + However, if you connect to a different repository, push history will be reset. +
+ + + + + +
+
+
+
+
diff --git a/frontend/src/components/ListSidebar.vue b/frontend/src/components/ListSidebar.vue
index 9fe47f9ee..f45b8263c 100644
--- a/frontend/src/components/ListSidebar.vue
+++ b/frontend/src/components/ListSidebar.vue
@@ -30,7 +30,7 @@
v-if="clearAllFilters"
type="button"
:class="{ disabled: isClearingFiltersLoading || isApplyingFiltersLoading }"
- class="btn btn-secondary mt-2 w-100"
+ class="btn btn-outline-gray mt-2 w-100"
v-on:click="isClearingFiltersLoading = true"
@click="clearAllFilters"
>
diff --git a/frontend/src/components/ReviewReminders.vue b/frontend/src/components/ReviewReminders.vue
index 36115c505..5dcb0254f 100644
--- a/frontend/src/components/ReviewReminders.vue
+++ b/frontend/src/components/ReviewReminders.vue
@@ -1,12 +1,20 @@
-
+
+
+
+ Your submitted models
+ Submit a new model +
+
+
+ Loading...
+
+
+ You have not submitted any models yet.
+
+
+
+
+
+
+ + +
-
+
+
Peer reviews must be requested on unpublished model releases so that you can continue to revise your submission - based on reviewer guidance. If you have already published your model - a new draft release will be created as an exact copy of your most recent - published release. + based on reviewer guidance.
+-
+
- + If you have already published this release, + a new draft will be created as an exact copy in order to allow for changes. + +
- + If this release was imported from GitHub and published, you will not be able to request a + review, and will need to create a new release on GitHub for peer review to take place on. + +
Once a model release passes peer review it will remain private until you choose to publish it,
at which point you may set a version number. Please note that
diff --git a/frontend/src/components/form/HoneypotField.vue b/frontend/src/components/form/HoneypotField.vue
index cd07c44a2..e07d62a1b 100644
--- a/frontend/src/components/form/HoneypotField.vue
+++ b/frontend/src/components/form/HoneypotField.vue
@@ -36,7 +36,7 @@ import FieldHelp from "@/components/form/FieldHelp.vue";
export interface HoneypotFieldProps {
name?: string;
- show: boolean;
+ show?: boolean;
}
const props = withDefaults(defineProps Connect your GitHub account with your CoMSES Net profile
+ Enter the name of your repository to connect it
+
+ This gives our service read and write access to repositories. Make sure to select the
+ individual repository(s) you want to connect, or allow access to all repositories (not
+ recommended).
+
+ This release does not conform to semantic versioning. Please specify a version number.
+
+ You'll need to create an empty repository on GitHub first. After you connect it, the
+ history will be built and you'll be able to push it.
+
+ Below is the current filesystem layout of the archival package as it was imported from GitHub.
+ Note that metadata files (
+ Please review the file tree to ensure that all necessary files are present and re-categorize
+ files as necessary. At least one code and documentation file is required for publishing.
+
+ If you do need to add, remove, or update any files, you can do so by
+ updating the source GitHub release
+ to point at a new tag with the desired changes, which will trigger a re-import of the release.
+ This can only be done if the release is unpublished or under review, once it is published the
+ file package is locked.
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/ConnectGitHubStep.vue b/frontend/src/components/githubIntegration/ConnectGitHubStep.vue
new file mode 100644
index 000000000..59e3eaff2
--- /dev/null
+++ b/frontend/src/components/githubIntegration/ConnectGitHubStep.vue
@@ -0,0 +1,63 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Connect GitHub
+
+
+
+
+ Make sure the CoMSES Integration GitHub App has access to the repository you want to
+ connect.
+ Manage permissions
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Released {{ (release.publishedAt || release.createdAt)?.split("T")[0] }}
+
+
+
+
+
+
+
+
+
+ Install App
+
+
+ Installed but not recognized? Try refreshing the page or
+
+ uninstalling and reinstalling the app .
+
+
+
+
+
+
+
+
+
+
+ Verify import and notify reviewers
+
+ Verify import
+
+
+
+
+
+
+
+ * Public model library releases indicated with
+
+ have been built into a git repository, each corresponding to a commit in a main branch,
+ version branch, and tag which can be pushed to the connected GitHub repository. We won't
+ overwrite any changes you make to the GitHub repository.
+
+
+
+ transferred
+ original
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/ReleaseSyncStatusCell.vue b/frontend/src/components/githubIntegration/ReleaseSyncStatusCell.vue
new file mode 100644
index 000000000..096459002
--- /dev/null
+++ b/frontend/src/components/githubIntegration/ReleaseSyncStatusCell.vue
@@ -0,0 +1,210 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Model Library releases
+
+
+
+
+
+
+
+
+
+ GitHub releases
+
+
+ No releases found. You must
+
+ create a release on GitHub
+ in order to import to the model library.
+
+
+
+
+
+ Fetching releases...
+
+
+
+ No releases available
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/githubIntegration/SelectSyncTypeStep.vue b/frontend/src/components/githubIntegration/SelectSyncTypeStep.vue
new file mode 100644
index 000000000..5309ee059
--- /dev/null
+++ b/frontend/src/components/githubIntegration/SelectSyncTypeStep.vue
@@ -0,0 +1,124 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {{ badgeText }}
+
+
+
+
+ update pending
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Connect to a repository you already have on GitHub, allowing you to
+ import releases made there to the CoMSES Model Library.
+
+
+
+
+
+
+
+ You will create a new GitHub repository, then a git repo will be automatically
+ constructed from your releases published here, which you can
+ push. This is an easy way to start using GitHub and you'll be able to
+ import any future releases created there.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Name the repository anything you want, but make sure it is public and
+ NOT initialized
+ with anything like a README, .gitignore, or license files.
+
+
+
+
+
+
+
+
+
+
-
-
@@ -79,6 +101,7 @@ const props = defineProps<{
reviewStatus: string;
isLive: boolean;
canEditOriginals: boolean;
+ isImported: boolean;
showPublishModal: boolean;
showUploadImageModal: boolean;
}>();
diff --git a/frontend/src/components/releaseEditor/FileTree.vue b/frontend/src/components/releaseEditor/FileTree.vue
index 59c9e99c4..682e8fc15 100644
--- a/frontend/src/components/releaseEditor/FileTree.vue
+++ b/frontend/src/components/releaseEditor/FileTree.vue
@@ -1,27 +1,108 @@
-
- {{ directory.label }}
+
+
+
diff --git a/frontend/src/components/releaseEditor/ImportedArchivePage.vue b/frontend/src/components/releaseEditor/ImportedArchivePage.vue
new file mode 100644
index 000000000..020b8b34d
--- /dev/null
+++ b/frontend/src/components/releaseEditor/ImportedArchivePage.vue
@@ -0,0 +1,76 @@
+
+
+
+ {{ directory.label }}
+
+
-
- {{ content.label }}
-
-
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/frontend/src/components/releaseEditor/ProgressSidebar.vue b/frontend/src/components/releaseEditor/ProgressSidebar.vue
index c9e87902a..f19247cfe 100644
--- a/frontend/src/components/releaseEditor/ProgressSidebar.vue
+++ b/frontend/src/components/releaseEditor/ProgressSidebar.vue
@@ -2,18 +2,18 @@
CITATION.cff, codemeta.json) are
+ automatically updated to contain the most accurate information about the model release and
+ reflect its location in the CoMSES Model Library.
+
+
+ Current Archival Package Filesystem Layout
+ Loading download preview...
+
+ {{ serverErrors.join(", ") }}
+
+
+
+