Support GremLite

.github/workflows/pise-publish.yml

@@ -103,6 +103,7 @@ jobs:
           tar -x -f pise-server-context.tar.gz -C pise-server-context
           tar -x -f pise-frontend-context.tar.gz -C pise-frontend-context
           tar -x -f pise-oca-context.tar.gz -C pise-oca-context
+          tar -x -f pise-oca-rg-context.tar.gz -C pise-oca-rg-context
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
@@ -138,5 +139,16 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ${{env.DH_NAMESPACE}}/pise:${{env.PISE_VERS}}
+            ${{env.DH_NAMESPACE}}/pise:${{ format('{0}-gl', env.PISE_VERS) }}
+      # Note: While we are publishing the RG version of the OCA, that is the one
+      # that gets the secondary tag ('latest' or 'edge'), for backwards-compatibility.
+      - if: ${{ steps.switch.outputs.pub-dh == 'yes' }}
+        name: Publish pise OCA-rg to Docker Hub
+        uses: docker/build-push-action@v4
+        with:
+          context: contexts/pise-oca-rg-context
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{env.DH_NAMESPACE}}/pise:${{ format('{0}-rg', env.PISE_VERS) }}
             ${{env.DH_NAMESPACE}}/pise:${{env.SECOND_TAG}}

changelog.d/99.improved.txt

@@ -0,0 +1,12 @@
+`pise/server` now supports the [GremLite](https://pypi.org/project/gremlite/)
+graph database system.
+
+FOR NOW: Two versions of the
+[PISE one-container app Docker image](https://hub.docker.com/r/proofscape/pise)
+will be offered, one using RedisGraph (tags ending in `-rg`), and one using
+GremLite (tags ending in `-gl`).
+
+FUTURE: The RedisGraph versions of the Docker image will eventually be dropped,
+and only GremLite versions will be produced. Users of RedisGraph versions of the
+app will have to rebuild all Proofscape repos in their library, as the GremLite
+versions will not be able to read the old graph database files.

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proofscape/pise-client",
-  "version": "0.31.0-dev",
+  "version": "0.30.4-dev",
   "license": "Apache-2.0",
   "description": "Client-side code for PISE, the Proofscape Integrated Study Environment",
   "repository": {

manage/conf/base_conf.py

@@ -176,11 +176,11 @@
 #
 # These are the default docker image tags that will be used, for various
 # services, or as starting points for building our own docker images.
-PYTHON_IMAGE_TAG = f'{PYTHON_VERSION}-slim-bullseye'
-REDIS_IMAGE_TAG = '6.2.1'
+PYTHON_IMAGE_TAG = f'{PYTHON_VERSION}-slim-bookworm'
+REDIS_IMAGE_TAG = '8.4.0-bookworm'
 REDISGRAPH_IMAGE_TAG = '6.2.6-v6'
 NEO4J_IMAGE_TAG = '4.4.46'
-GREMLIN_SERVER_IMAGE_TAG = '3.6.0'
+GREMLIN_SERVER_IMAGE_TAG = '3.7.5'
 JANUSGRAPH_IMAGE_TAG = '0.6.0'
 NGINX_IMAGE_TAG = '1.24.0'
 # If you want RedisInsight to be dispatched as part of the MCA when RedisGraph

manage/pise_python_vers.env

@@ -1,2 +1,2 @@
 # The Python version currently used in PISE is:
-PISE_PYTHON_VERS=3.11.13
+PISE_PYTHON_VERS=3.11.14

manage/tools/build.py

@@ -27,6 +27,7 @@
 import conf
 from manage import cli, PFSC_ROOT, PFSC_MANAGE_ROOT
 from conf import DOCKER_CMD, DOCKER_PLATFORM
+from tools.deploy.services import GdbCode
 import tools.license
 from tools.util import get_version_numbers
 import topics.pfsc.write_license_files as write_license_files
@@ -154,9 +155,9 @@ def write_dockerignore_for_pyc():
 def build_license_info_dict(skip_licensing: bool, tag: str) -> dict:
     skip_key = 'Deliberately skipping license file generation for development purposes.'
     if skip_licensing:
-        if not tag.endswith('-dev'):
+        if 'dev' not in tag.split('-'):
             raise click.UsageError(
-                'When using the --skip-licensing switch, your TAG must end with "-dev".\n'
+                'When using the --skip-licensing switch, your TAG must contain "-dev".\n'
                 'The purpose of the switch is to let you skip updating license info when\n'
                 'trying out new packages (or package upgrades) during development.'
             )
@@ -280,12 +281,15 @@ def oca_readiness_checks(release=False, client=True, client_min=False, pdf=True,
 
 
 @build.command()
+@click.option('--gdb',
+              default=GdbCode.GL, prompt='Graph database (gl, re)',
+              help='Choose the graph DB to be built into the image. gl=GremLite, re=RedisGraph')
 @click.option('--dump', is_flag=True, help="Dump Dockerfile to stdout before building.")
 @click.option('--dry-run', is_flag=True, help="Do not actually build; just print docker command.")
 @click.option('--tar-path', help="Instead of building, save the context tar file to this path.")
 @click.option('--skip-licensing', is_flag=True, default=False, help='Skip gathering of license info. For dev only.')
 @click.argument('tag')
-def oca(dump, dry_run, tar_path, skip_licensing, tag: str):
+def oca(gdb, dump, dry_run, tar_path, skip_licensing, tag: str):
     """
     Build a `pise` (one-container app) docker image, and give it a TAG.
 
@@ -294,6 +298,14 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str):
     content repos. It is configured in "personal server mode" and comes with
     RedisGraph as the GDB.
     """
+    if gdb not in [GdbCode.GL, GdbCode.RE]:
+        raise click.UsageError('Must select one of the graph databases GL (GremLite) or RE (RedisGraph).')
+
+    gdb_tag_segment_lookup = {GdbCode.GL: 'gl', GdbCode.RE: 'rg'}
+    segment = gdb_tag_segment_lookup[gdb]
+    if segment not in tag.split('-'):
+        raise click.UsageError(f'When using the {gdb} graph database, the tag must contain "-{segment}".')
+
     if not dry_run:
         oca_readiness_checks(client=True, client_min=False, pdf=True, pyodide=True, whl=True)
 
@@ -307,7 +319,7 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str):
     from topics.pfsc import write_oca_eula_file
     from topics.pfsc import write_worker_and_web_supervisor_ini
     from topics.pfsc import write_proofscape_oca_dockerfile
-    from topics.redis import write_redisgraph_ini
+    from topics.redis import write_redis_ini, write_redisgraph_ini
     with tempfile.TemporaryDirectory(dir=SRC_TMP_ROOT) as tmp_dir_name:
         with open(os.path.join(tmp_dir_name, 'license_info.json'), 'w') as f:
             f.write(json.dumps(license_info, indent=4))
@@ -322,14 +334,23 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str):
             ini = write_worker_and_web_supervisor_ini(
                 worker=False, web=True, use_venv=False, oca=True)
             f.write(ini)
-        with open(os.path.join(tmp_dir_name, 'redisgraph.ini'), 'w') as f:
-            ini = write_redisgraph_ini(use_conf_file=True)
-            f.write(ini)
+
+        if gdb == GdbCode.GL:
+            with open(os.path.join(tmp_dir_name, 'redis.ini'), 'w') as f:
+                ini = write_redis_ini(use_conf_file=True)
+                f.write(ini)
+        else:
+            assert gdb == GdbCode.RE
+            with open(os.path.join(tmp_dir_name, 'redisgraph.ini'), 'w') as f:
+                ini = write_redisgraph_ini(use_conf_file=True)
+                f.write(ini)
+
         with open(os.path.join(tmp_dir_name, 'oca_version.txt'), 'w') as f:
             f.write(tag)
+
         tmp_dir_rel_path = os.path.relpath(tmp_dir_name, start=SRC_ROOT)
         write_dockerignore_for_pyc()
-        df = write_proofscape_oca_dockerfile(tmp_dir_rel_path)
+        df = write_proofscape_oca_dockerfile(gdb, tmp_dir_rel_path)
         finalize(df, 'pise', tag, dump, dry_run, tar_path=tar_path)
 
 

manage/tools/deploy/__init__.py

@@ -98,8 +98,8 @@ def production(gdb, workers, demos, dump_dc, dirname, official, pfsc_tag):
 
 @deploy.command()
 @click.option('--gdb',
-              default=GdbCode.RE, prompt='Graph database (re, nj, tk, ja, np)',
-              help='List one or more graph DBs. re=RedisGraph, nj=Neo4j, tk=TinkerGraph, ja=JanusGraph, np=Neptune')
+              default=GdbCode.GL, prompt='Graph database (gl, re, nj, tk, ja, np)',
+              help='List one or more graph DBs. gl=GremLite, re=RedisGraph, nj=Neo4j, tk=TinkerGraph, ja=JanusGraph, np=Neptune')
 @click.option('--pfsc-tag', default='testing', prompt='pise-server image tag',
               help='Use `pise-server:TEXT` docker image.')
 @click.option('--frontend-tag',
@@ -130,7 +130,7 @@ def production(gdb, workers, demos, dump_dc, dirname, official, pfsc_tag):
 @click.option('--dummy', is_flag=True, help='Write a docker compose yml for a dummy deployment (Hello World web app).')
 @click.option('--lib-vol', help="A pre-existing docker volume to be mounted to /proofscape/lib")
 @click.option('--build-vol', help="A pre-existing docker volume to be mounted to /proofscape/build")
-@click.option('--gdb-vol', help="A pre-existing docker volume for the graph db. Only RedisGraph currently supported.")
+@click.option('--gdb-vol', help="A pre-existing docker volume for the graph db.")
 @click.option('--no-redis', is_flag=True, help='Only allowed when RedisGraph is sole GDB, which is then used in place of Redis.')
 def generate(gdb, pfsc_tag, frontend_tag, oca_tag, official, workers, demos, mount_code, mount_pkg, dump_dc,
              dirname, no_local, flask_config, per_deploy_dirs, static_redir, static_acao, dummy,
@@ -170,7 +170,7 @@ def generate(gdb, pfsc_tag, frontend_tag, oca_tag, official, workers, demos, mou
         raise click.UsageError(f'Legal GDB codes are: {", ".join(GdbCode.all)}')
     if len(gdb) > len(s):
         raise click.UsageError('Cannot repeat graph database selections.')
-    if no_redis and s != {'re'}:
+    if no_redis and s != {GdbCode.RE}:
         raise click.UsageError('RedisGraph must be sole GDB selection, when using --no-redis.')
 
     dirname_prefix = 'production_' if production_mode else None
@@ -180,7 +180,7 @@ def generate(gdb, pfsc_tag, frontend_tag, oca_tag, official, workers, demos, mou
 
     # admin shell script
     admin_sh_script = write_admin_sh_script(
-        new_dir_name, new_dir_path, pfsc_tag, flask_config,
+        new_dir_name, new_dir_path, pfsc_tag, flask_config, gdb,
         demos=demos, mount_code=mount_code, mount_pkg=mount_pkg,
         official=official, no_redis=no_redis
     )
@@ -597,7 +597,7 @@ def write_gdb_dot_env(d, gdb, uri_lookup_method, comment=False):
                 'out': i > 0 or GdbCode.requires_manual_URI(code),
             }
             if comment:
-                v['comment'] = GdbCode.service_name(code) + ":"
+                v['comment'] = GdbCode.comment_name(code) + ":"
             d[f"GRAPHDB_URI_{i}"] = v
         d['post-gdb-block'] = {'comment': ''}
 
@@ -673,14 +673,14 @@ def write_dc_script(deploy_dir_name):
 
 
 def write_admin_sh_script(
-        deploy_dir_name, deploy_dir_path, pfsc_tag, flask_config,
+        deploy_dir_name, deploy_dir_path, pfsc_tag, flask_config, gdb,
         demos=False, mount_code=False, mount_pkg=None,
         official=False, no_redis=False
     ):
     # Want all the same bind mounts that are used in a pfsc worker container,
     # so that admin can do anything a worker can do.
     d = services.pise_server(
-        deploy_dir_path, 'worker', flask_config, tag=pfsc_tag,
+        deploy_dir_path, 'worker', flask_config, gdb, tag=pfsc_tag,
         demos=demos, mount_code=mount_code, mount_pkg=mount_pkg,
         official=official, no_redis=no_redis
     )
@@ -742,10 +742,13 @@ def write_docker_compose_yaml(deploy_dir_name, deploy_dir_path, gdb, pfsc_tag, f
         svc_defn = writer(altdir=altdir)
 
         if gdb_vol:
-            # TODO: Provide support for use of named volumes with other GDBs besides RedisGraph
-            if code == GdbCode.RE:
+            data_vol_mount_pt = GdbCode.service_container_data_vol_mount_point(code)
+            if data_vol_mount_pt is None:
+                name = GdbCode.service_name(code)
+                raise click.UsageError(f'--gdb-vol switch is not currently supported for {name}')
+            else:
                 svc_defn['volumes'] = [
-                    f'{gdb_vol}:/data'
+                    f'{gdb_vol}:{data_vol_mount_pt}'
                 ]
 
         s_full[name] = svc_defn
@@ -765,10 +768,10 @@ def write_docker_compose_yaml(deploy_dir_name, deploy_dir_path, gdb, pfsc_tag, f
 
     s_app = {}
     def write_pfsc_service(cmd):
-        return services.pise_server(deploy_dir_path, cmd, flask_config,
-            tag=pfsc_tag, gdb=gdb, workers=workers, demos=demos,
+        return services.pise_server(deploy_dir_path, cmd, flask_config, gdb,
+            tag=pfsc_tag, workers=workers, demos=demos,
             mount_code=mount_code, mount_pkg=mount_pkg, official=official, altdir=altdir,
-            lib_vol=lib_vol, build_vol=build_vol, no_redis=no_redis)
+            lib_vol=lib_vol, build_vol=build_vol, gdb_vol=gdb_vol, no_redis=no_redis)
 
     for n in range(workers):
         svc_pfscwork = write_pfsc_service('worker')

manage/tools/deploy/services.py

@@ -22,6 +22,8 @@
 
 
 class GdbCode:
+    # GremLite
+    GL = 'gl'
     # RedisGraph
     RE = 're'
     # Neo4j
@@ -33,7 +35,7 @@ class GdbCode:
     # Neptune
     NP = 'np'
 
-    all = [RE, NJ, TK, JA, NP]
+    all = [GL, RE, NJ, TK, JA, NP]
 
     # Those that are deployed via container:
     via_container = [RE, NJ, TK, JA]
@@ -67,6 +69,12 @@ def standard_port(cls, code):
             cls.NP: 8182,
         }[code]
 
+    @classmethod
+    def comment_name(cls, code):
+        if code == cls.GL:
+            return 'GremLite'
+        return cls.service_name(code)
+
     @classmethod
     def service_name(cls, code):
         return {
@@ -89,13 +97,17 @@ def uri_path(cls, code):
 
     @classmethod
     def localhost_URI(cls, code):
-        if code == cls.NP:
+        if code == cls.GL:
+            return cls.GremLite_URI(PFSC_ROOT)
+        elif code == cls.NP:
             return cls.Neptune_URI()
         return f'{cls.protocol(code)}://localhost:{cls.host_port(code)}{cls.uri_path(code)}'
 
     @classmethod
     def docker_URI(cls, code):
-        if code == cls.NP:
+        if code == cls.GL:
+            return cls.GremLite_URI('/proofscape')
+        elif code == cls.NP:
             return cls.Neptune_URI()
         return f'{cls.protocol(code)}://{cls.service_name(code)}:{cls.standard_port(code)}{cls.uri_path(code)}'
 
@@ -108,6 +120,20 @@ def service_defn_writer(cls, code):
             cls.JA: janusgraph,
         }[code]
 
+    @classmethod
+    def service_container_data_vol_mount_point(cls, code):
+        """
+        Give the mount point within the service container where the GDB stores its data.
+        """
+        return {
+            cls.RE: '/data',
+        }.get(code, None)
+
+    @classmethod
+    def GremLite_URI(cls, pfsc_root_path):
+        db_file_path = pathlib.Path(pfsc_root_path) / 'graphdb/gl/gremlite.db'
+        return f'file://{db_file_path}'
+
     @classmethod
     def Neptune_URI(cls):
         prefix = conf.AWS_NEPTUNE_URI_PREFIX or 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
@@ -240,11 +266,11 @@ def get_proofscape_subdir_abs_fs_path_on_host(subdir_name, altdir=None):
         return resolve_pfsc_root_subdir(subdir_name)
 
 
-def pise_server(deploy_dir_path, mode, flask_config, tag='latest',
-                gdb=None, workers=1, demos=False,
+def pise_server(deploy_dir_path, mode, flask_config, gdb, tag='latest',
+                workers=1, demos=False,
                 mount_code=False, mount_pkg=None,
                 official=False, altdir=None,
-                lib_vol=None, build_vol=None,
+                lib_vol=None, build_vol=None, gdb_vol=None,
                 no_redis=False):
     d = {
         'image': f"{'proofscape/' if official else ''}pise-server:{tag}",
@@ -260,8 +286,10 @@ def pise_server(deploy_dir_path, mode, flask_config, tag='latest',
     }
 
     if no_redis:
+        # When this is set, gdb is required to be equal to [GdbCode.RE].
+        assert gdb == [GdbCode.RE]
         if mode == 'websrv':
-            # redisgraph will be added below, as GDB dependency
+            # redisgraph will be added below, as GDB dependency, so we don't add it here.
             pass
         else:
             d['depends_on'].append('redisgraph')
@@ -284,21 +312,36 @@ def pise_server(deploy_dir_path, mode, flask_config, tag='latest',
     }[mode]
     d['environment'][mode_env_var] = 1
 
-    gdb = gdb or [GdbCode.RE]
+    # Both web server and workers should wait for any GDB services to be ready.
+    d['depends_on'].extend(GdbCode.service_name(code) for code in gdb if code in GdbCode.via_container)
+
+    # Web server depends on workers.
     if mode == 'websrv':
-        d['depends_on'].extend(GdbCode.service_name(code) for code in gdb if code in GdbCode.via_container)
         d['depends_on'].extend([f'pfscwork{n}' for n in range(workers)])
+
+    # Both web server and all workers should have access to the same GremLite db file, if using GremLite.
+    if GdbCode.GL in gdb:
+        direc = 'graphdb/gl'
+        # It is only for this use case that this function accepts a `gdb_vol` kwarg.
+        volume = gdb_vol or get_proofscape_subdir_abs_fs_path_on_host(direc, altdir=altdir)
+        d['volumes'].append(f'{volume}:/proofscape/{direc}')
+
     if conf.EMAIL_TEMPLATE_DIR:
         d['volumes'].append(f'{resolve_fs_path("EMAIL_TEMPLATE_DIR")}:/home/pfsc/proofscape/src/_email_templates:ro')
+
+    # If mounting code, it goes into all web server and workers
     if mount_code:
         if demos:
             d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-demo-repos")}:/home/pfsc/demos:ro')
         d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-server/pfsc")}:/home/pfsc/proofscape/src/pfsc-server/pfsc:ro')
         d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-server/config.py")}:/home/pfsc/proofscape/src/pfsc-server/config.py:ro')
         d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-ise/package.json")}:/home/pfsc/proofscape/src/client/package.json:ro')
+
+    # Likewise for packages
     if mount_pkg:
         for pkg in [s.strip() for s in mount_pkg.split(',')]:
             d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-server/venv/lib/python3.8/site-packages")}/{pkg}:/usr/local/lib/python3.8/site-packages/{pkg}')
+
     return d
 
 

manage/tools/license.py

@@ -296,7 +296,8 @@ def gather_licensing_info(verbose=False):
     vers['python'] = get_python_version_for_images()
     vers['supervisor'] = pfsc_conf.SUPERVISOR_VERSION
     vers['redisgraph'] = pfsc_conf.REDISGRAPH_IMAGE_TAG
-    vers['redis-server'] = get_redis_server_version_for_oca()
+    vers['redis-server-rg-oca'] = pfsc_conf.REDISGRAPH_IMAGE_TAG.split('-')[0]
+    vers['redis-server-gl-oca'] = get_redis_server_version_for_oca()
     vers['nginx'] = pfsc_conf.NGINX_IMAGE_TAG
 
     # These are the one-off cases: