From f28447b28a16de6dcb1df99c128b9446d5a7daf3 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 8 Nov 2024 19:47:50 -0500 Subject: [PATCH 01/35] Docstring --- server/tests/trial_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/tests/trial_build.py b/server/tests/trial_build.py index c092232..23bb76e 100644 --- a/server/tests/trial_build.py +++ b/server/tests/trial_build.py @@ -29,6 +29,10 @@ existing indexing of test repos, and then builds the desired version plus all its dependencies, in topological order. +Setting TRIAL_BUILD_CLEAN=1 means that, if the version is WIP, we will set the +builder to erase any and all existing pickle files for the modules in this repo, +as well as to `make clean` with Sphinx (if a part of the build). + We use a script instead of a unit test because sometimes these builds can mess up other unit tests. In particular, our procedure when building at a numbered version messes them up by clearing indexing, and you must do From eddfa6a09d936638bb665e71a8fc9945a135fe27 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 8 Nov 2024 19:48:30 -0500 Subject: [PATCH 02/35] Add hint for profiling --- server/tests/util/build_repos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/tests/util/build_repos.py b/server/tests/util/build_repos.py index a26e357..e297a65 100644 --- a/server/tests/util/build_repos.py +++ b/server/tests/util/build_repos.py @@ -22,5 +22,9 @@ if __name__ == "__main__": #build_big() + build_all() + # To show timings: + #build_all(verbose=2) + build_at_wip() From 288f865a76b5047f1623e04b8bb7216359b1d3cf Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 8 Nov 2024 19:49:44 -0500 Subject: [PATCH 03/35] Repair `CypherGraphReader.num_edges_in_db()` This was double counting, due to matching each directed edge once in each direction. --- server/pfsc/gdb/cypher/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pfsc/gdb/cypher/reader.py b/server/pfsc/gdb/cypher/reader.py index ccbc3ae..0d01f0e 100644 --- a/server/pfsc/gdb/cypher/reader.py +++ b/server/pfsc/gdb/cypher/reader.py @@ -41,7 +41,7 @@ def num_nodes_in_db(self): return check_count(res) def num_edges_in_db(self): - res = self.session.run("MATCH ()-[e]-() RETURN count(e)") + res = self.session.run("MATCH ()-[e]->() RETURN count(e)") return check_count(res) def all_nodes_under_repo(self, repopath): From d5cf4dccf0f51376af4a32b888a07f53678dee5f Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 8 Nov 2024 19:52:36 -0500 Subject: [PATCH 04/35] Refactor for better profiling --- server/pfsc/gdb/gremlin/indexing.py | 54 ++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/server/pfsc/gdb/gremlin/indexing.py b/server/pfsc/gdb/gremlin/indexing.py index bfb6712..69efbbf 100644 --- a/server/pfsc/gdb/gremlin/indexing.py +++ b/server/pfsc/gdb/gremlin/indexing.py @@ -105,9 +105,13 @@ def ix002682(mii, gtx, N=12): return new_targeting_relns -def ix002681(mii, gtx): +def ix002681_FULL(mii, gtx): """ Make a separate traversal for each new node and edge. + + This function ("_FULL") does both vertices and edges; + see `ix002681()` which does those in separate functions, + which can be more informative for profiling. """ lp_maj_to_db_uid = {} if mii.V_add: @@ -147,6 +151,54 @@ def ix002681(mii, gtx): return new_targeting_relns +def ix002681(mii, gtx): + lp_maj_to_db_uid = ix002681V(mii, gtx) + return ix002681E(mii, gtx, lp_maj_to_db_uid) + + +def ix002681V(mii, gtx): + lp_maj_to_db_uid = {} + if mii.V_add: + mii.note_begin_indexing_phase(260) + kNodes = [mii.get_kNode(uid) for uid in mii.V_add] + for u in kNodes: + tr = gtx.add_v(u.node_type).as_('v') + for k, v in u.get_property_dict().items(): + tr = tr.property(k, v) + db_uid = tr.select('v').by(__.id_()).next() + lp_maj_to_db_uid[(u.libpath, u.major)] = db_uid + mii.note_task_element_completed(260) + return lp_maj_to_db_uid + + +def ix002681E(mii, gtx, lp_maj_to_db_uid): + new_targeting_relns = [] + if mii.E_add: + mii.note_begin_indexing_phase(280) + kRelns = [mii.get_kReln(uid) for uid in mii.E_add] + for r in kRelns: + if r.reln_type == IndexType.TARGETS: + new_targeting_relns.append(r) + head_id = lp_maj_to_db_uid.get((r.head_libpath, r.head_major)) + if head_id is None: + tr = lp_covers(r.head_libpath, r.head_major, + gtx.V().has_label(r.head_type)) + else: + tr = gtx.V(head_id) + tr = tr.as_('head') + tail_id = lp_maj_to_db_uid.get((r.tail_libpath, r.tail_major)) + if tail_id is None: + tr = lp_covers(r.tail_libpath, r.tail_major, + tr.V().has_label(r.tail_type)) + else: + tr = tr.V(tail_id) + tr = tr.add_e(r.reln_type).to('head') + tr = set_kReln_reln_props(r, tr) + tr.iterate() + mii.note_task_element_completed(280) + return new_targeting_relns + + def ix002680(mii, gtx): """ Try forming all new nodes and edges in a single traversal. From fae1e602217803220170b496585f44ad16f18b73 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 8 Nov 2024 20:09:37 -0500 Subject: [PATCH 05/35] Add support for sqlite-based GDB * As a part of this, we take long-overdue steps to put *all* methods of the GDB writer classes into transactions. Formerly, just the `index_module()` method (plus one method in the cypher writer) used transactions. * We also address an issue arising with gremlinpython 3.6.1. We were compatible only up to 3.6.0, but now can go up to 3.7.2 (latest at this time). --- server/config.py | 10 ++- server/pfsc/gdb/__init__.py | 39 ++++++++++-- server/pfsc/gdb/cypher/writer.py | 83 +++++++++++------------- server/pfsc/gdb/gremlin/writer.py | 45 +++++++++---- server/pfsc/gdb/writer.py | 101 +++++++++++++++++++++++++++--- 5 files changed, 204 insertions(+), 74 deletions(-) diff --git a/server/config.py b/server/config.py index cff9bca..9bc8a5f 100644 --- a/server/config.py +++ b/server/config.py @@ -308,9 +308,13 @@ class Config: GDB_USERNAME = os.getenv("GDB_USERNAME") or '' GDB_PASSWORD = os.getenv("GDB_PASSWORD") or '' - # Some GDB systems support transactions, some do not. If we can tell based - # on the GRAPHDB_URI (such as RedisGraph versus Neo4j) then we ignore this - # variable; if we cannot (such as with a Gremlin URI) then we follow this. + # For certain, known GDB systems, whether we use transactions or not is fixed and + # not configurable. For Neo4j we use them (because it supports them), for RedisGraph + # we do not (because it doesn't), and for gremlite we do, because it doesn't work + # properly otherwise. + # + # For any other GDB system you might use, you can control whether transactions are + # used or not, using this config var. USE_TRANSACTIONS = bool(int(os.getenv("USE_TRANSACTIONS", 0))) # NOTE: math job timeouts are only relevant if you are performing math jobs diff --git a/server/pfsc/gdb/__init__.py b/server/pfsc/gdb/__init__.py index 9397b4f..ac79a0c 100644 --- a/server/pfsc/gdb/__init__.py +++ b/server/pfsc/gdb/__init__.py @@ -19,6 +19,8 @@ from gremlin_python.process.anonymous_traversal import traversal from gremlin_python.process.graph_traversal import GraphTraversalSource from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection +from gremlin_python.driver.serializer import GraphSONSerializersV3d0 +from gremlite import SQLiteConnection from wsc_grempy_transport.transport import websocket_client_transport_factory import neo4j @@ -31,14 +33,13 @@ from pfsc.gdb.cypher.rg import RedisGraphWrapper from pfsc.gdb.gremlin.reader import GremlinGraphReader -from pfsc.gdb.gremlin.writer import GremlinGraphWriter +from pfsc.gdb.gremlin.writer import GremlinGraphWriter, GREMLIN_REMOTE_NAME from pfsc.gdb.gremlin.util import GtxTx_Gts GDB_OBJECT_NAME = "gdb" GRAPH_READER_NAME = "graph_reader" GRAPH_WRITER_NAME = "graph_writer" -GREMLIN_REMOTE_NAME = "gremlin_remote" def get_gdb(): @@ -48,14 +49,36 @@ def get_gdb(): if GDB_OBJECT_NAME not in flask_g: uri = current_app.config["GRAPHDB_URI"] # Decide by the form of the URI which graph database system we are using. - if uri.endswith('/gremlin'): - remote = DriverRemoteConnection( - uri, transport_factory=websocket_client_transport_factory) + protocol = uri.split(":")[0] + if uri.endswith('/gremlin') or protocol == 'file': + if protocol == 'file': + path = uri[7:] + # We ignore pise/server's `USE_TRANSACTIONS` config var and never allow + # sqlite to work in its native autocommit mode, because it is horribly slow. + remote = SQLiteConnection(path, autocommit=False) + else: + # Starting with gremlinpython==3.6.1, we have to explicitly request the + # `GraphSONSerializersV3d0` message serializer. This was the default in 3.6.0, + # but in 3.6.1 they changed the default to `GraphBinarySerializersV1`. + # See https://tinkerpop.apache.org/docs/current/upgrade/#_tinkerpop_3_6_1 + # + # One consequence of the change is that, while the `E()` step still requires IDs + # to be passed as instances of `gremlin_python.statics.long`, the 3.6.1 serializer + # *reports* IDs to you as `int`, whereas 3.6.0 reported them as `long`. + # + # Our code sometimes stores edge IDs as returned from certain Gremlin queries, and + # then attempts to pass these same objects right back as arguments to `E()` steps. + # (For example this happens in `pfsc.gdb.gremlin.writer.GremlinGraphWriter.ix0200()`.) + # In order for this to work from gremlinpython 3.6.1 and onward, without having to + # explicitly recast the IDs as longs, we need to use the old serializer. + remote = DriverRemoteConnection( + uri, transport_factory=websocket_client_transport_factory, + message_serializer=GraphSONSerializersV3d0() + ) # Store the remote so it can be closed later. setattr(flask_g, GREMLIN_REMOTE_NAME, remote) gdb = traversal(GtxTx_Gts).with_remote(remote) else: - protocol = uri.split(":")[0] if protocol in ['redis', 'rediss']: gdb = RedisGraphWrapper(uri) elif protocol in ['bolt', 'neo4j']: @@ -94,6 +117,10 @@ def get_graph_writer() -> GraphWriter: use_transactions = current_app.config["USE_TRANSACTIONS"] writer = GremlinGraphWriter(reader, use_transactions) else: + # Here we're not consulting the USE_TRANSACTIONS config var at all, + # because we've only contemplated two GDBs that use Cypher, namely, + # Neo4j and RedisGraph. With the former we automatically use transactions (since + # it supports them), and with the latter we do not (because it doesn't). writer = CypherGraphWriter(reader) setattr(flask_g, GRAPH_WRITER_NAME, writer) return getattr(flask_g, GRAPH_WRITER_NAME) diff --git a/server/pfsc/gdb/cypher/writer.py b/server/pfsc/gdb/cypher/writer.py index 100a1a4..1292693 100644 --- a/server/pfsc/gdb/cypher/writer.py +++ b/server/pfsc/gdb/cypher/writer.py @@ -27,9 +27,13 @@ class CypherGraphWriter(GraphWriter): def __init__(self, reader): super().__init__(reader) - self.session = self.gdb.session() + self._session = self.gdb.session() # TODO: Maybe the gdb teardown needs to close the session? + @property + def session(self): + return self._tx if self._tx is not None else self._session + def new_transaction(self): return self.session.begin_transaction() @@ -149,7 +153,7 @@ def ix0400(self, mii, tx): props=mii.write_version_node_props() ) - def clear_test_indexing(self): + def _clear_test_indexing(self): self.session.run(f""" MATCH (u) WHERE u.repopath STARTS WITH 'test.' OPTIONAL MATCH (u)-[:{IndexType.BUILD}]->(b) @@ -164,7 +168,7 @@ def _do_delete_all_under_repo(self, repopath): DETACH DELETE u, b """, repopath=repopath) - def delete_full_build_at_version(self, repopath, version=pfsc.constants.WIP_TAG): + def _delete_full_build_at_version(self, repopath, version=pfsc.constants.WIP_TAG): M, m, p = get_padded_components(version) self.session.run(f""" MATCH (u {{repopath: $repopath}}) @@ -181,7 +185,7 @@ def _add_user(self, username, j_props): SET u.properties = $j_props """, username=username, j_props=j_props) - def delete_user(self, username, *, + def _delete_user(self, username, *, definitely_want_to_delete_this_user=False): if not definitely_want_to_delete_this_user: return 0 @@ -192,7 +196,7 @@ def delete_user(self, username, *, info = res.consume() return info.counters.nodes_deleted - def delete_all_notes_of_one_user(self, username, *, + def _delete_all_notes_of_one_user(self, username, *, definitely_want_to_delete_all_notes=False): if not definitely_want_to_delete_all_notes: return @@ -207,48 +211,35 @@ def _update_user(self, username, j_props): SET u.properties = $j_props """, username=username, j_props=j_props) - def record_user_notes(self, username, user_notes): + def _record_user_notes(self, username, user_notes): major0 = self.reader.adaptall(user_notes.goal_major) - # It's important that we structure this as a transaction, for the case - # of the user of the one-container app on their own machine. There we - # use RedisGraph, and it's only a call to `commit_transaction()` that - # prompts our `RedisGraphWrapper` class to dump to disk. The user's - # notes should always be persisted to disk as soon as they're recorded - # in the GDB. In other cases -- say, Neo4j in a production setting -- - # structuring as a transaction does no harm. - tx = self.new_transaction() - try: - res = tx.run(f""" - MATCH (g {{libpath: $goalpath, major: $major}}) RETURN id(g) - """, goalpath=user_notes.goalpath, major=major0) - rec = res.single() - if rec is None: - raise PfscExcep(f'Cannot record notes. Origin {user_notes.write_origin()} does not exist.') - goal_db_id = rec.value() - - if user_notes.is_blank(): - tx.run(f""" - MATCH (u:{IndexType.USER} {{username: $username}})-[e:{IndexType.NOTES}]->(g) - WHERE ID(g) = $goal_db_id - DELETE e - """, username=username, goal_db_id=goal_db_id) - else: - tx.run(f""" - MATCH (u:{IndexType.USER} {{username: $username}}), (g) - WHERE ID(g) = $goal_db_id - MERGE (u)-[e:{IndexType.NOTES}]->(g) - SET e.state = $state - SET e.notes = $notes - """, username=username, goal_db_id=goal_db_id, state=user_notes.state, notes=user_notes.notes) - except: - self.rollback_transaction(tx) - raise + tx = self.session + res = tx.run(f""" + MATCH (g {{libpath: $goalpath, major: $major}}) RETURN id(g) + """, goalpath=user_notes.goalpath, major=major0) + rec = res.single() + if rec is None: + raise PfscExcep(f'Cannot record notes. Origin {user_notes.write_origin()} does not exist.') + goal_db_id = rec.value() + + if user_notes.is_blank(): + tx.run(f""" + MATCH (u:{IndexType.USER} {{username: $username}})-[e:{IndexType.NOTES}]->(g) + WHERE ID(g) = $goal_db_id + DELETE e + """, username=username, goal_db_id=goal_db_id) else: - self.commit_transaction(tx) + tx.run(f""" + MATCH (u:{IndexType.USER} {{username: $username}}), (g) + WHERE ID(g) = $goal_db_id + MERGE (u)-[e:{IndexType.NOTES}]->(g) + SET e.state = $state + SET e.notes = $notes + """, username=username, goal_db_id=goal_db_id, state=user_notes.state, notes=user_notes.notes) # ---------------------------------------------------------------------- - def record_module_source(self, modpath, version, modtext): + def _record_module_source(self, modpath, version, modtext): major0 = self.reader.adaptall(version) self.session.run(f""" MATCH (m:{IndexType.MODULE} {{libpath: $modpath}}) @@ -258,13 +249,13 @@ def record_module_source(self, modpath, version, modtext): SET s.pfsc = $modtext """, modpath=modpath, major=major0, version=version, modtext=modtext) - def record_repo_manifest(self, repopath, version, manifest_json): + def _record_repo_manifest(self, repopath, version, manifest_json): self.session.run(f""" MATCH (v:{IndexType.VERSION} {{repopath: $repopath, version: $version}}) SET v.manifest = $manifest_json """, repopath=repopath, version=version, manifest_json=manifest_json) - def record_dashgraph(self, deducpath, version, dg_json): + def _record_dashgraph(self, deducpath, version, dg_json): major0 = self.reader.adaptall(version) self.session.run(f""" MATCH (d:{IndexType.DEDUC} {{libpath: $deducpath}}) @@ -274,7 +265,7 @@ def record_dashgraph(self, deducpath, version, dg_json): SET db.json = $dg_json """, deducpath=deducpath, major=major0, version=version, dg_json=dg_json) - def record_annobuild(self, annopath, version, anno_html, anno_json): + def _record_annobuild(self, annopath, version, anno_html, anno_json): major0 = self.reader.adaptall(version) self.session.run(f""" MATCH (a:{IndexType.ANNO} {{libpath: $annopath}}) @@ -285,7 +276,7 @@ def record_annobuild(self, annopath, version, anno_html, anno_json): SET ab.json = $anno_json """, annopath=annopath, major=major0, version=version, anno_html=anno_html, anno_json=anno_json) - def delete_builds_under_module(self, modpath, version): + def _delete_builds_under_module(self, modpath, version): self.session.run(f""" MATCH (u {{modpath: $modpath}})-[b:{IndexType.BUILD}]->(v) WHERE b.{IndexType.P_BUILD_VERS} = $version diff --git a/server/pfsc/gdb/gremlin/writer.py b/server/pfsc/gdb/gremlin/writer.py index 08523f1..53b136e 100644 --- a/server/pfsc/gdb/gremlin/writer.py +++ b/server/pfsc/gdb/gremlin/writer.py @@ -14,8 +14,10 @@ # limitations under the License. # # --------------------------------------------------------------------------- # +from flask import g as flask_g from gremlin_python.process.graph_traversal import __ from gremlin_python.process.traversal import TextP +from gremlite import SQLiteConnection from pfsc.constants import WIP_TAG, IndexType from pfsc.gdb.writer import GraphWriter @@ -29,6 +31,17 @@ from pfsc.excep import PfscExcep +GREMLIN_REMOTE_NAME = "gremlin_remote" + + +def using_sqlite(): + """ + Check whether we are using SQLite. + """ + remote = getattr(flask_g, GREMLIN_REMOTE_NAME) + return isinstance(remote, SQLiteConnection) + + class GremlinGraphWriter(GraphWriter): """ GraphWriter that speaks Gremlin. @@ -38,11 +51,19 @@ class GremlinGraphWriter(GraphWriter): def __init__(self, reader, use_transactions=True): super().__init__(reader) - self.use_transactions = use_transactions + + # If we are using an SQLiteConnection, then we force use of transactions, + # since otherwise we actually don't function properly. (This is probably something + # to fix in the future -- we have largely relied on GDB systems like RedisGraph that + # do autocommit throughout, so the issue simply doesn't arise. And we cannot similarly + # use autocommit with SQLiteConnection, since it is horribly slow.) + self.use_transactions = True if using_sqlite() else use_transactions + + self._tx = None @property def g(self) -> GTX: - return self.gdb + return self._tx if self._tx is not None else self.gdb def new_transaction(self): return self.g.tx().begin() if self.use_transactions else self.g @@ -155,7 +176,7 @@ def ix0400(self, mii, gtx): 'version': mii.version }, mii.write_version_node_props()).iterate() - def clear_test_indexing(self): + def _clear_test_indexing(self): self.g.V().has('repopath', TextP.starting_with('test.')).union( __.identity(), __.out(IndexType.BUILD), @@ -168,7 +189,7 @@ def _do_delete_all_under_repo(self, repopath): __.out(IndexType.BUILD), ).barrier().drop().iterate() - def delete_full_build_at_version(self, repopath, version=WIP_TAG): + def _delete_full_build_at_version(self, repopath, version=WIP_TAG): M, m, p = get_padded_components(version) self.g.V().has('repopath', repopath).or_( __.has('version', version), @@ -189,7 +210,7 @@ def _add_user(self, username, j_props): 'properties': j_props }, label_order=1).iterate() - def delete_user(self, username, *, + def _delete_user(self, username, *, definitely_want_to_delete_this_user=False): if not definitely_want_to_delete_this_user: return 0 @@ -200,7 +221,7 @@ def delete_user(self, username, *, c1 = is_user(username, self.g.V()).count().next() return c0 - c1 - def delete_all_notes_of_one_user(self, username, *, + def _delete_all_notes_of_one_user(self, username, *, definitely_want_to_delete_all_notes=False): if not definitely_want_to_delete_all_notes: return @@ -212,7 +233,7 @@ def _update_user(self, username, j_props): __.property('properties', j_props) ).iterate() - def record_user_notes(self, username, user_notes): + def _record_user_notes(self, username, user_notes): major0 = self.reader.adaptall(user_notes.goal_major) is_goal = lambda tr: lp_maj(user_notes.goalpath, major0, tr) @@ -247,7 +268,7 @@ def record_user_notes(self, username, user_notes): # ---------------------------------------------------------------------- - def record_module_source(self, modpath, version, modtext): + def _record_module_source(self, modpath, version, modtext): major0 = self.reader.adaptall(version) lp_covers(modpath, major0, self.g.V()).as_('m') \ .add_v(IndexType.MOD_SRC) \ @@ -255,14 +276,14 @@ def record_module_source(self, modpath, version, modtext): .add_e(IndexType.BUILD).from_('m') \ .property(IndexType.P_BUILD_VERS, version).iterate() - def record_repo_manifest(self, repopath, version, manifest_json): + def _record_repo_manifest(self, repopath, version, manifest_json): self.g.V().has_label(IndexType.VERSION) \ .has('repopath', repopath).has('version', version).union( __.properties('manifest').drop(), __.property('manifest', manifest_json) ).iterate() - def record_dashgraph(self, deducpath, version, dg_json): + def _record_dashgraph(self, deducpath, version, dg_json): major0 = self.reader.adaptall(version) lp_covers(deducpath, major0, self.g.V()).as_('d') \ .add_v(IndexType.DEDUC_BUILD) \ @@ -270,7 +291,7 @@ def record_dashgraph(self, deducpath, version, dg_json): .add_e(IndexType.BUILD).from_('d') \ .property(IndexType.P_BUILD_VERS, version).iterate() - def record_annobuild(self, annopath, version, anno_html, anno_json): + def _record_annobuild(self, annopath, version, anno_html, anno_json): major0 = self.reader.adaptall(version) lp_covers(annopath, major0, self.g.V()).as_('a') \ .add_v(IndexType.ANNO_BUILD) \ @@ -279,7 +300,7 @@ def record_annobuild(self, annopath, version, anno_html, anno_json): .add_e(IndexType.BUILD).from_('a') \ .property(IndexType.P_BUILD_VERS, version).iterate() - def delete_builds_under_module(self, modpath, version): + def _delete_builds_under_module(self, modpath, version): self.g.V().has('modpath', modpath) \ .out_e(IndexType.BUILD).has(IndexType.P_BUILD_VERS, version) \ .in_v().drop().iterate() diff --git a/server/pfsc/gdb/writer.py b/server/pfsc/gdb/writer.py index f489d57..d7f5676 100644 --- a/server/pfsc/gdb/writer.py +++ b/server/pfsc/gdb/writer.py @@ -14,6 +14,8 @@ # limitations under the License. # # --------------------------------------------------------------------------- # +from contextlib import contextmanager +import functools import json import pfsc.constants @@ -23,12 +25,23 @@ from pfsc.gdb.user import User, make_new_user_properties_dict +def as_transaction(): + def decor(func): + @functools.wraps(func) + def wrapper(graph_writer, *args, **kwargs): + with graph_writer.embed_new_transaction(): + return func(graph_writer, *args, **kwargs) + return wrapper + return decor + + class GraphWriter: """Abstract base class for graph database writers. """ def __init__(self, reader): self.gdb = reader.gdb self._reader = reader + self._tx = None @property def reader(self) -> GraphReader: @@ -38,6 +51,22 @@ def new_transaction(self): """Start a new transaction. """ raise NotImplementedError + @contextmanager + def embed_new_transaction(self): + if self._tx is None: + self._tx = tx = self.new_transaction() + try: + yield tx + except Exception as e: + self.rollback_transaction(tx) + raise e from None + else: + self.commit_transaction(tx) + finally: + self._tx = None + else: + yield self._tx + def commit_transaction(self, tx): """Commit a transaction. """ raise NotImplementedError @@ -67,17 +96,12 @@ def index_module(self, mii): msg = f'Release `{mii.version}` of repo `{mii.repopath}`' \ ' has already been indexed.' raise PfscExcep(msg, PECode.ATTEMPTED_RELEASE_REINDEX) - tx = self.new_transaction() - try: + + with self.embed_new_transaction() as tx: self.ix0100(mii, tx) new_targeting_relns = self.ix0200(mii, tx) self.ix0300(mii, new_targeting_relns, tx) self.ix0400(mii, tx) - except Exception as e: - self.rollback_transaction(tx) - raise e from None - else: - self.commit_transaction(tx) def clear_wip_indexing(self, mii, tx): """ @@ -157,12 +181,30 @@ def ix0400(self, mii, tx): """ raise NotImplementedError +# -------------------v + + """ + As a general pattern in this abstract base class, every method has a counterpart + whose name is the same except with a leading underscore. The second method is + abstract, and is to be implemented by the concrete subclasses; the first method + sometimes has some work to do, but other times serves only to receive the + `@as_transaction` decorator, and invoke the abstract method. (We do it this way + because if you simply decorate an abstract method, the decorator has no effect + on the concrete implementations. We could decorate the latter, but then we'd have + to repeat ourselves.) + """ + + @as_transaction() def clear_test_indexing(self): """ Clear all indexing under the `test` repo family. """ + return self._clear_test_indexing() + + def _clear_test_indexing(self): raise NotImplementedError + @as_transaction() def delete_everything_under_repo(self, repopath): """ Delete all nodes and edges under a given repopath, at all versions. @@ -181,12 +223,14 @@ def _do_delete_all_under_repo(self, repopath): """ raise NotImplementedError + @as_transaction() def delete_full_wip_build(self, repopath): """ Delete everything for a given repo @WIP. """ self.delete_full_build_at_version(repopath, version=pfsc.constants.WIP_TAG) + @as_transaction() def delete_full_build_at_version(self, repopath, version=pfsc.constants.WIP_TAG): """ Delete everything for a given repo at a given version. @@ -195,10 +239,14 @@ def delete_full_build_at_version(self, repopath, version=pfsc.constants.WIP_TAG) mode, then deleting any but the latest numbered version for a given repo will result in an inconsistent state in the index. """ + return self._delete_full_build_at_version(repopath, version=version) + + def _delete_full_build_at_version(self, repopath, version=pfsc.constants.WIP_TAG): raise NotImplementedError # ---------------------------------------------------------------------- + @as_transaction() def add_user(self, username, usertype, email, orgs_owned_by_user): """ Add a new user. @@ -220,6 +268,7 @@ def add_user(self, username, usertype, email, orgs_owned_by_user): def _add_user(self, username, j_props): raise NotImplementedError + @as_transaction() def merge_user(self, username, usertype, email, orgs_owned_by_user): """ Load a user if they already exist; otherwise add them as new. @@ -237,6 +286,7 @@ def merge_user(self, username, usertype, email, orgs_owned_by_user): user.update_owned_orgs(orgs_owned_by_user) return user, is_new + @as_transaction() def delete_user(self, username, *, definitely_want_to_delete_this_user=False): """ @@ -247,8 +297,13 @@ def delete_user(self, username, *, @param definitely_want_to_delete_this_user: bool, a programming check @return: int, being the number of User nodes deleted (always 0 or 1) """ + return self._delete_user(username, definitely_want_to_delete_this_user=definitely_want_to_delete_this_user) + + def _delete_user(self, username, *, + definitely_want_to_delete_this_user=False): raise NotImplementedError + @as_transaction() def delete_all_notes_of_one_user(self, username, *, definitely_want_to_delete_all_notes=False): """ @@ -257,8 +312,15 @@ def delete_all_notes_of_one_user(self, username, *, @param username: the user whose notes are to be deleted. @param definitely_want_to_delete_all_notes: bool, a programming check """ + return self._delete_all_notes_of_one_user( + username, + definitely_want_to_delete_all_notes=definitely_want_to_delete_all_notes) + + def _delete_all_notes_of_one_user(self, username, *, + definitely_want_to_delete_all_notes=False): raise NotImplementedError + @as_transaction() def update_user(self, user): """ Update the properties of an existing user. @@ -272,6 +334,7 @@ def update_user(self, user): def _update_user(self, username, j_props): raise NotImplementedError + @as_transaction() def record_user_notes(self, username, user_notes): """ Record a user's notes on a goal. @@ -282,10 +345,14 @@ def record_user_notes(self, username, user_notes): @param username: str, the user @param user_notes: UserNotes to be recorded """ + return self._record_user_notes(username, user_notes) + + def _record_user_notes(self, username, user_notes): raise NotImplementedError # ---------------------------------------------------------------------- + @as_transaction() def record_module_source(self, modpath, version, modtext): """ Record the pfsc source code for a given module at a given version. @@ -294,8 +361,12 @@ def record_module_source(self, modpath, version, modtext): @param version: the version of the module @param modtext: the pfsc source code of the module """ + return self._record_module_source(modpath, version, modtext) + + def _record_module_source(self, modpath, version, modtext): raise NotImplementedError + @as_transaction() def record_repo_manifest(self, repopath, version, manifest_json): """ Record the JSON for a given repo's manifest, at a given version. @@ -304,8 +375,12 @@ def record_repo_manifest(self, repopath, version, manifest_json): @param version: the version of the repo @param manifest_json: the JSON of the repo's manifest """ + return self._record_repo_manifest(repopath, version, manifest_json) + + def _record_repo_manifest(self, repopath, version, manifest_json): raise NotImplementedError + @as_transaction() def record_dashgraph(self, deducpath, version, dg_json): """ Record the JSON for a given deduc's dashgraph, at a given version. @@ -314,8 +389,12 @@ def record_dashgraph(self, deducpath, version, dg_json): @param version: the version of the deduc @param dg_json: the JSON of the deduc's dashgraph """ + return self._record_dashgraph(deducpath, version, dg_json) + + def _record_dashgraph(self, deducpath, version, dg_json): raise NotImplementedError + @as_transaction() def record_annobuild(self, annopath, version, anno_html, anno_json): """ Record the HTML and JSON for a given anno, at a given version. @@ -325,8 +404,12 @@ def record_annobuild(self, annopath, version, anno_html, anno_json): @param anno_html: the anno's HTML @param anno_json: the anno's JSON """ + return self._record_annobuild(annopath, version, anno_html, anno_json) + + def _record_annobuild(self, annopath, version, anno_html, anno_json): raise NotImplementedError + @as_transaction() def delete_builds_under_module(self, modpath, version): """ Delete all edges and nodes representing built products, under a given @@ -337,10 +420,14 @@ def delete_builds_under_module(self, modpath, version): @param modpath: the libpath of the module @param version: the version of the module """ + return self._delete_builds_under_module(modpath, version) + + def _delete_builds_under_module(self, modpath, version): raise NotImplementedError # ---------------------------------------------------------------------- + @as_transaction() def set_approval(self, widgetpath, version, approved): """ This provides a mechanism for reviewing display code on `disp` widgets From ad423e1fb4ba91d0cd032afb57214b417fcd4136 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 8 Nov 2024 20:09:46 -0500 Subject: [PATCH 06/35] Add comments --- server/pfsc/gdb/gremlin/reader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/pfsc/gdb/gremlin/reader.py b/server/pfsc/gdb/gremlin/reader.py index 4cb8880..381a77a 100644 --- a/server/pfsc/gdb/gremlin/reader.py +++ b/server/pfsc/gdb/gremlin/reader.py @@ -320,6 +320,11 @@ def _extract_goal_infos_and_load_user_notes(self, username, tr): infos = tr.coalesce( __.has('origin'), + # Note: Here we are using pattern matching using a `where()` step. + # https://kelvinlawrence.net/book/Gremlin-Graph-Guide.html#patternwhere + # This means the `as_()` step inside the `where()` does not assign a + # label, but instead requires the object to equal the one that previously + # received that label, outside of the `where()` step. __.in_e(IndexType.NOTES).where( __.out_v().as_('u') ) From 0c990886a4845b7cb5f4f8ef52b11f1469438944 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Wed, 13 Nov 2024 07:57:35 -0500 Subject: [PATCH 07/35] Improve comments --- server/config.py | 30 ++++++++++++++++++++++++------ server/pfsc/gdb/__init__.py | 16 +++++++++------- server/pfsc/gdb/gremlin/writer.py | 7 ++----- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/server/config.py b/server/config.py index 9bc8a5f..1f163d7 100644 --- a/server/config.py +++ b/server/config.py @@ -308,13 +308,31 @@ class Config: GDB_USERNAME = os.getenv("GDB_USERNAME") or '' GDB_PASSWORD = os.getenv("GDB_PASSWORD") or '' - # For certain, known GDB systems, whether we use transactions or not is fixed and - # not configurable. For Neo4j we use them (because it supports them), for RedisGraph - # we do not (because it doesn't), and for gremlite we do, because it doesn't work - # properly otherwise. + # Some GDB systems support transactions, allowing us to control atomicity by grouping + # changes together before committing them, while others do not. The others must work + # instead in an "autocommit" mode, where all changes are immediately committed. # - # For any other GDB system you might use, you can control whether transactions are - # used or not, using this config var. + # Where transactions are supported, they are to be preferred. They provide better + # atomicity, and in some cases may make operations significantly faster (for example, + # GremLite is much, much faster when we group our changes using transactions). + # + # The current design in pise/server is influenced by the set of GDB systems against + # which it has ever been tested. Among Cypher systems, there are only two: Neo4j, + # and RedisGraph. The former supports transactions, the latter does not. As a consequence, + # pise/server will always use transactions when using Cypher, unless connecting to RedisGraph. + # Thus, the `USE_TRANSACTIONS` config var currently has no effect when connecting to any + # Cypher GDB. + # + # If you are trying to use pise/server with a Cypher GDB other than RedisGraph that does + # not support transactions, please open an issue at the PISE GitHub page. + # + # On the Gremlin side, we have tested against GremLite, where transactions are supported, + # and other systems such as TinkerGraph where they are not. With GremLite, we always + # use transactions, and the `USE_TRANSACTIONS` config var again has no effect. + # + # It is only with other Gremlin systems, besides GremLite, where `USE_TRANSACTIONS` + # controls anything. So if you are using a Gremlin system other than GremLite, and it + # does support transactions, then you should set this to 1. USE_TRANSACTIONS = bool(int(os.getenv("USE_TRANSACTIONS", 0))) # NOTE: math job timeouts are only relevant if you are performing math jobs diff --git a/server/pfsc/gdb/__init__.py b/server/pfsc/gdb/__init__.py index ac79a0c..fab8432 100644 --- a/server/pfsc/gdb/__init__.py +++ b/server/pfsc/gdb/__init__.py @@ -53,8 +53,9 @@ def get_gdb(): if uri.endswith('/gremlin') or protocol == 'file': if protocol == 'file': path = uri[7:] - # We ignore pise/server's `USE_TRANSACTIONS` config var and never allow - # sqlite to work in its native autocommit mode, because it is horribly slow. + # As can be seen in the `GremlinGraphWriter.__init__()` method, we will + # ignore pise/server's `USE_TRANSACTIONS` config var and will always use + # transactions with GremLite. Therefore here we want to turn off its autocommit mode. remote = SQLiteConnection(path, autocommit=False) else: # Starting with gremlinpython==3.6.1, we have to explicitly request the @@ -62,15 +63,16 @@ def get_gdb(): # but in 3.6.1 they changed the default to `GraphBinarySerializersV1`. # See https://tinkerpop.apache.org/docs/current/upgrade/#_tinkerpop_3_6_1 # - # One consequence of the change is that, while the `E()` step still requires IDs - # to be passed as instances of `gremlin_python.statics.long`, the 3.6.1 serializer - # *reports* IDs to you as `int`, whereas 3.6.0 reported them as `long`. + # One consequence of the change is that, while TinkerGraph still uses longs (not ints) + # as edge IDs, so that the `E()` step still requires IDs to be passed as instances + # of `gremlin_python.statics.long`, the 3.6.1 serializer *reports* edge IDs to you + # as `int` (whereas 3.6.0 reported them as `long`). # # Our code sometimes stores edge IDs as returned from certain Gremlin queries, and # then attempts to pass these same objects right back as arguments to `E()` steps. # (For example this happens in `pfsc.gdb.gremlin.writer.GremlinGraphWriter.ix0200()`.) - # In order for this to work from gremlinpython 3.6.1 and onward, without having to - # explicitly recast the IDs as longs, we need to use the old serializer. + # In order for this to work with TinkerGraph from gremlinpython 3.6.1 and onward, + # without having to explicitly recast the IDs as longs, we need to use the old serializer. remote = DriverRemoteConnection( uri, transport_factory=websocket_client_transport_factory, message_serializer=GraphSONSerializersV3d0() diff --git a/server/pfsc/gdb/gremlin/writer.py b/server/pfsc/gdb/gremlin/writer.py index 53b136e..93dca7c 100644 --- a/server/pfsc/gdb/gremlin/writer.py +++ b/server/pfsc/gdb/gremlin/writer.py @@ -52,11 +52,8 @@ class GremlinGraphWriter(GraphWriter): def __init__(self, reader, use_transactions=True): super().__init__(reader) - # If we are using an SQLiteConnection, then we force use of transactions, - # since otherwise we actually don't function properly. (This is probably something - # to fix in the future -- we have largely relied on GDB systems like RedisGraph that - # do autocommit throughout, so the issue simply doesn't arise. And we cannot similarly - # use autocommit with SQLiteConnection, since it is horribly slow.) + # If we are using an SQLiteConnection, then we force use of transactions, since + # it supports them and they are to be preferred (both for atomicity and increased speed). self.use_transactions = True if using_sqlite() else use_transactions self._tx = None From ef8044a6dd4ba2424f3c14d74401ce00133cbed8 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Wed, 13 Nov 2024 10:20:37 -0500 Subject: [PATCH 08/35] Add help for logging --- server/pfsc/gdb/__init__.py | 6 +++++- server/pytest.ini | 6 ++++++ server/tests/util/build_repos.py | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/server/pfsc/gdb/__init__.py b/server/pfsc/gdb/__init__.py index fab8432..dc31844 100644 --- a/server/pfsc/gdb/__init__.py +++ b/server/pfsc/gdb/__init__.py @@ -53,10 +53,14 @@ def get_gdb(): if uri.endswith('/gremlin') or protocol == 'file': if protocol == 'file': path = uri[7:] + # Set these True to log low level SQLite usage: + log_plans = False + check_qqc_patterns = False # As can be seen in the `GremlinGraphWriter.__init__()` method, we will # ignore pise/server's `USE_TRANSACTIONS` config var and will always use # transactions with GremLite. Therefore here we want to turn off its autocommit mode. - remote = SQLiteConnection(path, autocommit=False) + remote = SQLiteConnection(path, autocommit=False, + log_plans=log_plans, check_qqc_patterns=check_qqc_patterns) else: # Starting with gremlinpython==3.6.1, we have to explicitly request the # `GraphSONSerializersV3d0` message serializer. This was the default in 3.6.0, diff --git a/server/pytest.ini b/server/pytest.ini index 4e261e6..2006a18 100644 --- a/server/pytest.ini +++ b/server/pytest.ini @@ -10,3 +10,9 @@ filterwarnings = ignore::DeprecationWarning:jinja2.*: ignore::DeprecationWarning:eventlet.*: ignore::DeprecationWarning:dns.*: + +# Uncomment to record test logs to file +#log_file = pytest.log +#log_cli_level = INFO +#log_file_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +#log_file_date_format = %Y-%m-%d %H:%M:%S diff --git a/server/tests/util/build_repos.py b/server/tests/util/build_repos.py index e297a65..288e3b7 100644 --- a/server/tests/util/build_repos.py +++ b/server/tests/util/build_repos.py @@ -21,6 +21,10 @@ from tests.util import build_all, build_at_wip, build_big if __name__ == "__main__": + # Uncomment to record logs: + #import logging + #logging.basicConfig(filename='build_repos.log', level=logging.INFO) + #build_big() build_all() From 70bdd9ff8a79dc9db74ec36f9b7bb440c03ba9dc Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 04:11:10 -0500 Subject: [PATCH 09/35] Add gremlite to reqs * Accordingly upgrade gremlinpython * For simplicity's sake, I also now give up on excluding the unneeded recursive reqs of gremlinpython. The purpose of that was to keep the docker images smaller, but that ship sailed when we accepted sphinx's inclusion of babel (30MB to format dates! -- see 230101 log notes) * I also turn off pip-compile annotations in the secondary reqs files. For some reason they now include *absolute* filesystem paths when referring to other reqs files named under `-c` options, instead of relative, and I don't like that. --- .github/workflows/pise-build-and-test.yml | 2 +- manage/conf/base_conf.py | 2 +- server/req/compile.sh | 9 +- server/req/dev-requirements.txt | 55 +- server/req/requirements.in | 15 +- server/req/requirements.nodeps | 5 +- server/req/requirements.txt | 716 +++++++++++++++++++++- server/req/test-requirements.txt | 43 +- 8 files changed, 724 insertions(+), 123 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index f996423..d2f1a77 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -48,7 +48,7 @@ jobs: ports: - "6381:6379" tinkergraph: - image: "tinkerpop/gremlin-server:3.6.0" + image: "tinkerpop/gremlin-server:3.7.5" ports: - "8182:8182" strategy: diff --git a/manage/conf/base_conf.py b/manage/conf/base_conf.py index 2e0ff20..bc3da37 100644 --- a/manage/conf/base_conf.py +++ b/manage/conf/base_conf.py @@ -180,7 +180,7 @@ REDIS_IMAGE_TAG = '6.2.1' REDISGRAPH_IMAGE_TAG = '6.2.6-v6' NEO4J_IMAGE_TAG = '4.4.46' -GREMLIN_SERVER_IMAGE_TAG = '3.6.0' +GREMLIN_SERVER_IMAGE_TAG = '3.7.5' JANUSGRAPH_IMAGE_TAG = '0.6.0' NGINX_IMAGE_TAG = '1.24.0' # If you want RedisInsight to be dispatched as part of the MCA when RedisGraph diff --git a/server/req/compile.sh b/server/req/compile.sh index e0e7e23..2f7bb59 100755 --- a/server/req/compile.sh +++ b/server/req/compile.sh @@ -3,5 +3,10 @@ # Run this script any time you update any of the *.in files. pip-compile --generate-hashes requirements.in -pip-compile --generate-hashes test-requirements.in -pip-compile --generate-hashes dev-requirements.in +# We're saying `--no-annotate` on the files that have `-c` constraints +# in them, since it seems pip-compile has begun to put *absolute* filesystem +# paths for the constraint files so named, in its annotations. I don't want +# the annotations to contain traces of the development machine where the +# work happened to be done. +pip-compile --generate-hashes --no-annotate test-requirements.in +pip-compile --generate-hashes --no-annotate dev-requirements.in diff --git a/server/req/dev-requirements.txt b/server/req/dev-requirements.txt index 0ec0082..3bef769 100644 --- a/server/req/dev-requirements.txt +++ b/server/req/dev-requirements.txt @@ -2,53 +2,32 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --generate-hashes dev-requirements.in +# pip-compile --generate-hashes --no-annotate dev-requirements.in # arrow==1.4.0 \ --hash=sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205 \ --hash=sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7 - # via rq-dashboard blinker==1.9.0 \ --hash=sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf \ --hash=sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc - # via - # -c requirements.txt - # flask build==1.4.0 \ --hash=sha256:6a07c1b8eb6f2b311b96fcbdbce5dab5fe637ffda0fd83c9cac622e927501596 \ --hash=sha256:f1b91b925aa322be454f8330c6fb48b465da993d1e7e7e6fa35027ec49f3c936 - # via pip-tools click==8.3.1 \ --hash=sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a \ --hash=sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6 - # via - # -c requirements.txt - # flask - # pip-tools - # rq flask==3.1.2 \ --hash=sha256:bf656c15c80190ed628ad08cdfd3aaa35beb087855e2f494910aa3774cc4fd87 \ --hash=sha256:ca1d8112ec8a6158cc29ea4858963350011b5c846a414cdb7a954aa9e967d03c - # via - # -c requirements.txt - # rq-dashboard invoke==2.2.1 \ --hash=sha256:2413bc441b376e5cd3f55bb5d364f973ad8bdd7bf87e53c79de3c11bf3feecc8 \ --hash=sha256:515bf49b4a48932b79b024590348da22f39c4942dff991ad1fb8b8baea1be707 - # via -r dev-requirements.in itsdangerous==2.2.0 \ --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 - # via - # -c requirements.txt - # flask jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 - # via - # -c requirements.txt - # -c test-requirements.txt - # flask markupsafe==3.0.3 \ --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ @@ -139,70 +118,38 @@ markupsafe==3.0.3 \ --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 - # via - # -c requirements.txt - # -c test-requirements.txt - # flask - # jinja2 - # werkzeug packaging==26.0 \ --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 - # via - # -c requirements.txt - # -c test-requirements.txt - # build - # wheel pip-tools==7.5.2 \ --hash=sha256:2d64d72da6a044da1110257d333960563d7a4743637e8617dd2610ae7b82d60f \ --hash=sha256:2fe16db727bbe5bf28765aeb581e792e61be51fc275545ef6725374ad720a1ce - # via -r dev-requirements.in pyproject-hooks==1.2.0 \ --hash=sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8 \ --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 - # via - # build - # pip-tools python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 - # via arrow redis==3.5.3 \ --hash=sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2 \ --hash=sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24 - # via - # -c requirements.txt - # rq - # rq-dashboard rq==1.8.0 \ --hash=sha256:2a253385bec82f0e723ad25fb547d3892900a94b3c3acc53514c4bd74897cb69 \ --hash=sha256:d861f1e794b595ae83f7b919d0c56d7102bc18f4f7b71395446d60d70344577c - # via - # -c requirements.txt - # rq-dashboard rq-dashboard==0.6.1 \ --hash=sha256:95163f0a8413e6be277d60eada0125f1c4b230577c3fa538a31a77a2cb556f8a - # via -r dev-requirements.in six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 - # via - # -c requirements.txt - # python-dateutil tzdata==2025.3 \ --hash=sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1 \ --hash=sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7 - # via arrow werkzeug==3.1.5 \ --hash=sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc \ --hash=sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67 - # via - # -c requirements.txt - # flask wheel==0.46.3 \ --hash=sha256:4b399d56c9d9338230118d705d9737a2a468ccca63d5e813e2a4fc7815d8bc4d \ --hash=sha256:e3e79874b07d776c40bd6033f8ddf76a7dad46a7b8aa1b2787a83083519a1803 - # via pip-tools # WARNING: The following packages were not pinned, but pip requires them to be # pinned when the requirements file includes hashes and the requirement is not diff --git a/server/req/requirements.in b/server/req/requirements.in index c2198ac..d8db83c 100644 --- a/server/req/requirements.in +++ b/server/req/requirements.in @@ -24,18 +24,9 @@ eventlet==0.40.4 # We need to talk to graph databases. neo4j==4.4.13 redisgraph==2.4.4 -# We want the `gremlinpython` package, but don't want to install all of its -# dependencies, since our use of `wsc-grempy-transport` obviates a substantial -# number of them. So we list these in the `requirements.nodeps` file instead. -# However, this means we must list here those of their dependencies that we do -# want to install. -# NOTE: Any time you upgrade `gremlinpython`, you must review its latest -# dependencies, and update this list accordingly. -# For gremlinpython==3.6.0: -aenum>=1.4.5,<4.0.0 -isodate>=0.6.0,<1.0.0 -# For wsc-grempy-transport==0.1.0: -websocket-client==1.3.2 +gremlinpython==3.7.5 +wsc-grempy-transport==0.1.0 +gremlite==0.1.1 # Lark parses all of our basic languages, such as the pfsc module syntax, # and the Meson proof script language. diff --git a/server/req/requirements.nodeps b/server/req/requirements.nodeps index 7c360ea..e1326d5 100644 --- a/server/req/requirements.nodeps +++ b/server/req/requirements.nodeps @@ -4,7 +4,4 @@ # you to apply the `--no-deps` option on a per-requirement basis; it applies # to a whole run, or not at all. -# NOTE: If you upgrade gremlinpython, you must review its latest dependencies, -# and update the corresponding list in the `requirements.in` file accordingly. -gremlinpython==3.6.0 -wsc-grempy-transport==0.1.0 +# Nothing here for now... diff --git a/server/req/requirements.txt b/server/req/requirements.txt index 6f549dc..184961b 100644 --- a/server/req/requirements.txt +++ b/server/req/requirements.txt @@ -8,11 +8,149 @@ aenum==3.1.16 \ --hash=sha256:7810cbb6b4054b7654e5a7bafbe16e9ee1d25ef8e397be699f63f2f3a5800433 \ --hash=sha256:9035092855a98e41b66e3d0998bd7b96280e85ceb3a04cc035636138a1943eaf \ --hash=sha256:bfaf9589bdb418ee3a986d85750c7318d9d2839c1b1a1d6fe8fc53ec201cf140 - # via -r requirements.in + # via gremlinpython +aiohappyeyeballs==2.6.1 \ + --hash=sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558 \ + --hash=sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 + # via aiohttp +aiohttp==3.13.3 \ + --hash=sha256:01ad2529d4b5035578f5081606a465f3b814c542882804e2e8cda61adf5c71bf \ + --hash=sha256:042e9e0bcb5fba81886c8b4fbb9a09d6b8a00245fd8d88e4d989c1f96c74164c \ + --hash=sha256:05861afbbec40650d8a07ea324367cb93e9e8cc7762e04dd4405df99fa65159c \ + --hash=sha256:084911a532763e9d3dd95adf78a78f4096cd5f58cdc18e6fdbc1b58417a45423 \ + --hash=sha256:0add0900ff220d1d5c5ebbf99ed88b0c1bbf87aa7e4262300ed1376a6b13414f \ + --hash=sha256:0db318f7a6f065d84cb1e02662c526294450b314a02bd9e2a8e67f0d8564ce40 \ + --hash=sha256:10b47b7ba335d2e9b1239fa571131a87e2d8ec96b333e68b2a305e7a98b0bae2 \ + --hash=sha256:1449ceddcdbcf2e0446957863af03ebaaa03f94c090f945411b61269e2cb5daf \ + --hash=sha256:147e422fd1223005c22b4fe080f5d93ced44460f5f9c105406b753612b587821 \ + --hash=sha256:1cb93e166e6c28716c8c6aeb5f99dfb6d5ccf482d29fe9bf9a794110e6d0ab64 \ + --hash=sha256:215a685b6fbbfcf71dfe96e3eba7a6f58f10da1dfdf4889c7dd856abe430dca7 \ + --hash=sha256:2712039939ec963c237286113c68dbad80a82a4281543f3abf766d9d73228998 \ + --hash=sha256:27234ef6d85c914f9efeb77ff616dbf4ad2380be0cda40b4db086ffc7ddd1b7d \ + --hash=sha256:28e027cf2f6b641693a09f631759b4d9ce9165099d2b5d92af9bd4e197690eea \ + --hash=sha256:2b8d8ddba8f95ba17582226f80e2de99c7a7948e66490ef8d947e272a93e9463 \ + --hash=sha256:2ba0eea45eb5cc3172dbfc497c066f19c41bac70963ea1a67d51fc92e4cf9a80 \ + --hash=sha256:2be0e9ccf23e8a94f6f0650ce06042cefc6ac703d0d7ab6c7a917289f2539ad4 \ + --hash=sha256:2e41b18a58da1e474a057b3d35248d8320029f61d70a37629535b16a0c8f3767 \ + --hash=sha256:2eb752b102b12a76ca02dff751a801f028b4ffbbc478840b473597fc91a9ed43 \ + --hash=sha256:2fc82186fadc4a8316768d61f3722c230e2c1dcab4200d52d2ebdf2482e47592 \ + --hash=sha256:2fff83cfc93f18f215896e3a190e8e5cb413ce01553901aca925176e7568963a \ + --hash=sha256:31a83ea4aead760dfcb6962efb1d861db48c34379f2ff72db9ddddd4cda9ea2e \ + --hash=sha256:34749271508078b261c4abb1767d42b8d0c0cc9449c73a4df494777dc55f0687 \ + --hash=sha256:34bac00a67a812570d4a460447e1e9e06fae622946955f939051e7cc895cfab8 \ + --hash=sha256:37239e9f9a7ea9ac5bf6b92b0260b01f8a22281996da609206a84df860bc1261 \ + --hash=sha256:37da61e244d1749798c151421602884db5270faf479cf0ef03af0ff68954c9dd \ + --hash=sha256:3b61b7169ababd7802f9568ed96142616a9118dd2be0d1866e920e77ec8fa92a \ + --hash=sha256:3d9908a48eb7416dc1f4524e69f1d32e5d90e3981e4e37eb0aa1cd18f9cfa2a4 \ + --hash=sha256:3dd4dce1c718e38081c8f35f323209d4c1df7d4db4bab1b5c88a6b4d12b74587 \ + --hash=sha256:4021b51936308aeea0367b8f006dc999ca02bc118a0cc78c303f50a2ff6afb91 \ + --hash=sha256:40c5e40ecc29ba010656c18052b877a1c28f84344825efa106705e835c28530f \ + --hash=sha256:425c126c0dc43861e22cb1c14ba4c8e45d09516d0a3ae0a3f7494b79f5f233a3 \ + --hash=sha256:44531a36aa2264a1860089ffd4dce7baf875ee5a6079d5fb42e261c704ef7344 \ + --hash=sha256:48e377758516d262bde50c2584fc6c578af272559c409eecbdd2bae1601184d6 \ + --hash=sha256:49a03727c1bba9a97d3e93c9f93ca03a57300f484b6e935463099841261195d3 \ + --hash=sha256:4ae5b5a0e1926e504c81c5b84353e7a5516d8778fbbff00429fe7b05bb25cbce \ + --hash=sha256:4e239d501f73d6db1522599e14b9b321a7e3b1de66ce33d53a765d975e9f4808 \ + --hash=sha256:56339a36b9f1fc708260c76c87e593e2afb30d26de9ae1eb445b5e051b98a7a1 \ + --hash=sha256:568f416a4072fbfae453dcf9a99194bbb8bdeab718e08ee13dfa2ba0e4bebf29 \ + --hash=sha256:5b179331a481cb5529fca8b432d8d3c7001cb217513c94cd72d668d1248688a3 \ + --hash=sha256:5b6073099fb654e0a068ae678b10feff95c5cae95bbfcbfa7af669d361a8aa6b \ + --hash=sha256:5d2d94f1f5fcbe40838ac51a6ab5704a6f9ea42e72ceda48de5e6b898521da51 \ + --hash=sha256:5dff64413671b0d3e7d5918ea490bdccb97a4ad29b3f311ed423200b2203e01c \ + --hash=sha256:5e1d8c8b8f1d91cd08d8f4a3c2b067bfca6ec043d3ff36de0f3a715feeedf926 \ + --hash=sha256:5f8ca7f2bb6ba8348a3614c7918cc4bb73268c5ac2a207576b7afea19d3d9f64 \ + --hash=sha256:642f752c3eb117b105acbd87e2c143de710987e09860d674e068c4c2c441034f \ + --hash=sha256:65d2ccb7eabee90ce0503c17716fc77226be026dcc3e65cce859a30db715025b \ + --hash=sha256:693781c45a4033d31d4187d2436f5ac701e7bbfe5df40d917736108c1cc7436e \ + --hash=sha256:694976222c711d1d00ba131904beb60534f93966562f64440d0c9d41b8cdb440 \ + --hash=sha256:697753042d57f4bf7122cab985bf15d0cef23c770864580f5af4f52023a56bd6 \ + --hash=sha256:69c56fbc1993fa17043e24a546959c0178fe2b5782405ad4559e6c13975c15e3 \ + --hash=sha256:6de499a1a44e7de70735d0b39f67c8f25eb3d91eb3103be99ca0fa882cdd987d \ + --hash=sha256:6fc0e2337d1a4c3e6acafda6a78a39d4c14caea625124817420abceed36e2415 \ + --hash=sha256:75ca857eba4e20ce9f546cd59c7007b33906a4cd48f2ff6ccf1ccfc3b646f279 \ + --hash=sha256:7a4a94eb787e606d0a09404b9c38c113d3b099d508021faa615d70a0131907ce \ + --hash=sha256:7b5e8fe4de30df199155baaf64f2fcd604f4c678ed20910db8e2c66dc4b11603 \ + --hash=sha256:7bfdc049127717581866fa4708791220970ce291c23e28ccf3922c700740fdc0 \ + --hash=sha256:7e63f210bc1b57ef699035f2b4b6d9ce096b5914414a49b0997c839b2bd2223c \ + --hash=sha256:7f9120f7093c2a32d9647abcaf21e6ad275b4fbec5b55969f978b1a97c7c86bf \ + --hash=sha256:8057c98e0c8472d8846b9c79f56766bcc57e3e8ac7bfd510482332366c56c591 \ + --hash=sha256:80dd4c21b0f6237676449c6baaa1039abae86b91636b6c91a7f8e61c87f89540 \ + --hash=sha256:81e97251d9298386c2b7dbeb490d3d1badbdc69107fb8c9299dd04eb39bddc0e \ + --hash=sha256:82611aeec80eb144416956ec85b6ca45a64d76429c1ed46ae1b5f86c6e0c9a26 \ + --hash=sha256:8542f41a62bcc58fc7f11cf7c90e0ec324ce44950003feb70640fc2a9092c32a \ + --hash=sha256:859bd3f2156e81dd01432f5849fc73e2243d4a487c4fd26609b1299534ee1845 \ + --hash=sha256:87797e645d9d8e222e04160ee32aa06bc5c163e8499f24db719e7852ec23093a \ + --hash=sha256:87b9aab6d6ed88235aa2970294f496ff1a1f9adcd724d800e9b952395a80ffd9 \ + --hash=sha256:8a60e60746623925eab7d25823329941aee7242d559baa119ca2b253c88a7bd6 \ + --hash=sha256:90455115e5da1c3c51ab619ac57f877da8fd6d73c05aacd125c5ae9819582aba \ + --hash=sha256:90751b8eed69435bac9ff4e3d2f6b3af1f57e37ecb0fbeee59c0174c9e2d41df \ + --hash=sha256:947c26539750deeaee933b000fb6517cc770bbd064bad6033f1cff4803881e43 \ + --hash=sha256:96d604498a7c782cb15a51c406acaea70d8c027ee6b90c569baa6e7b93073679 \ + --hash=sha256:988a8c5e317544fdf0d39871559e67b6341065b87fceac641108c2096d5506b7 \ + --hash=sha256:9a9dc347e5a3dc7dfdbc1f82da0ef29e388ddb2ed281bfce9dd8248a313e62b7 \ + --hash=sha256:9ae8dd55c8e6c4257eae3a20fd2c8f41edaea5992ed67156642493b8daf3cecc \ + --hash=sha256:9af5e68ee47d6534d36791bbe9b646d2a7c7deb6fc24d7943628edfbb3581f29 \ + --hash=sha256:9b174f267b5cfb9a7dba9ee6859cecd234e9a681841eb85068059bc867fb8f02 \ + --hash=sha256:9bf9f7a65e7aa20dd764151fb3d616c81088f91f8df39c3893a536e279b4b984 \ + --hash=sha256:9d4c940f02f49483b18b079d1c27ab948721852b281f8b015c058100e9421dd1 \ + --hash=sha256:9ebf57d09e131f5323464bd347135a88622d1c0976e88ce15b670e7ad57e4bd6 \ + --hash=sha256:a19884d2ee70b06d9204b2727a7b9f983d0c684c650254679e716b0b77920632 \ + --hash=sha256:a1e53262fd202e4b40b70c3aff944a8155059beedc8a89bba9dc1f9ef06a1b56 \ + --hash=sha256:a2212ad43c0833a873d0fb3c63fa1bacedd4cf6af2fee62bf4b739ceec3ab239 \ + --hash=sha256:a45530014d7a1e09f4a55f4f43097ba0fd155089372e105e4bff4ca76cb1b168 \ + --hash=sha256:a949eee43d3782f2daae4f4a2819b2cb9b0c5d3b7f7a927067cc84dafdbb9f88 \ + --hash=sha256:add1da70de90a2569c5e15249ff76a631ccacfe198375eead4aadf3b8dc849dc \ + --hash=sha256:af71fff7bac6bb7508956696dce8f6eec2bbb045eceb40343944b1ae62b5ef11 \ + --hash=sha256:b04be762396457bef43f3597c991e192ee7da460a4953d7e647ee4b1c28e7046 \ + --hash=sha256:b0d95340658b9d2f11d9697f59b3814a9d3bb4b7a7c20b131df4bcef464037c0 \ + --hash=sha256:b1a6102b4d3ebc07dad44fbf07b45bb600300f15b552ddf1851b5390202ea2e3 \ + --hash=sha256:b46020d11d23fe16551466c77823df9cc2f2c1e63cc965daf67fa5eec6ca1877 \ + --hash=sha256:b556c85915d8efaed322bf1bdae9486aa0f3f764195a0fb6ee962e5c71ef5ce1 \ + --hash=sha256:b903a4dfee7d347e2d87697d0713be59e0b87925be030c9178c5faa58ea58d5c \ + --hash=sha256:b928f30fe49574253644b1ca44b1b8adbd903aa0da4b9054a6c20fc7f4092a25 \ + --hash=sha256:b99281b0704c103d4e11e72a76f1b543d4946fea7dd10767e7e1b5f00d4e5704 \ + --hash=sha256:bae5c2ed2eae26cc382020edad80d01f36cb8e746da40b292e68fec40421dc6a \ + --hash=sha256:bb4f7475e359992b580559e008c598091c45b5088f28614e855e42d39c2f1033 \ + --hash=sha256:bbe7d4cecacb439e2e2a8a1a7b935c25b812af7a5fd26503a66dadf428e79ec1 \ + --hash=sha256:bfc1cc2fe31a6026a8a88e4ecfb98d7f6b1fec150cfd708adbfd1d2f42257c29 \ + --hash=sha256:c014c7ea7fb775dd015b2d3137378b7be0249a448a1612268b5a90c2d81de04d \ + --hash=sha256:c048058117fd649334d81b4b526e94bde3ccaddb20463a815ced6ecbb7d11160 \ + --hash=sha256:c0e2d366af265797506f0283487223146af57815b388623f0357ef7eac9b209d \ + --hash=sha256:c19b90316ad3b24c69cd78d5c9b4f3aa4497643685901185b65166293d36a00f \ + --hash=sha256:c685f2d80bb67ca8c3837823ad76196b3694b0159d232206d1e461d3d434666f \ + --hash=sha256:c6b8568a3bb5819a0ad087f16d40e5a3fb6099f39ea1d5625a3edc1e923fc538 \ + --hash=sha256:d32764c6c9aafb7fb55366a224756387cd50bfa720f32b88e0e6fa45b27dcf29 \ + --hash=sha256:d5a372fd5afd301b3a89582817fdcdb6c34124787c70dbcc616f259013e7eef7 \ + --hash=sha256:d60ac9663f44168038586cab2157e122e46bdef09e9368b37f2d82d354c23f72 \ + --hash=sha256:dca68018bf48c251ba17c72ed479f4dafe9dbd5a73707ad8d28a38d11f3d42af \ + --hash=sha256:de2c184bb1fe2cbd2cefba613e9db29a5ab559323f994b6737e370d3da0ac455 \ + --hash=sha256:e3531d63d3bdfa7e3ac5e9b27b2dd7ec9df3206a98e0b3445fa906f233264c57 \ + --hash=sha256:e50a2e1404f063427c9d027378472316201a2290959a295169bcf25992d04558 \ + --hash=sha256:e636b3c5f61da31a92bf0d91da83e58fdfa96f178ba682f11d24f31944cdd28c \ + --hash=sha256:ea37047c6b367fd4bd632bff8077449b8fa034b69e812a18e0132a00fae6e808 \ + --hash=sha256:f33ed1a2bf1997a36661874b017f5c4b760f41266341af36febaf271d179f6d7 \ + --hash=sha256:f76c1e3fe7d7c8afad7ed193f89a292e1999608170dcc9751a7462a87dfd5bc0 \ + --hash=sha256:f9444f105664c4ce47a2a7171a2418bce5b7bae45fb610f4e2c36045d85911d3 \ + --hash=sha256:fc290605db2a917f6e81b0e1e0796469871f5af381ce15c604a3c5c7e51cb730 \ + --hash=sha256:fc353029f176fd2b3ec6cfc71be166aba1936fe5d73dd1992ce289ca6647a9aa \ + --hash=sha256:fee0c6bc7db1de362252affec009707a17478a00ec69f797d23ca256e36d5940 + # via gremlinpython +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp alabaster==0.7.16 \ --hash=sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65 \ --hash=sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92 # via sphinx +async-timeout==4.0.3 \ + --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \ + --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028 + # via gremlinpython +attrs==25.4.0 \ + --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ + --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 + # via aiohttp babel==2.18.0 \ --hash=sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d \ --hash=sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35 @@ -277,6 +415,140 @@ flask-socketio==5.6.0 \ --hash=sha256:42a7bc552013633875ad320e39462323b4f7334594f1658d72b6ffed99940d4c \ --hash=sha256:894ad031d9440ca3fad388dd301ca33d13b301a2563933ca608d30979ef0a7c1 # via -r requirements.in +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal furo==2022.12.7 \ --hash=sha256:7cb76c12a25ef65db85ab0743df907573d03027a33631f17d267e598ebb191f7 \ --hash=sha256:d8008f8efbe7587a97ba533c8b2df1f9c21ee9b3e5cad0d27f61193d38b1a986 @@ -336,6 +608,17 @@ greenlet==3.3.1 \ --hash=sha256:e84b51cbebf9ae573b5fbd15df88887815e3253fc000a7d0ff95170e8f7e9729 \ --hash=sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53 # via eventlet +gremlinpython==3.7.5 \ + --hash=sha256:37f132622299c7def3d104a91e0522df397d6b98d5c0c43f9efdbe080b86dc49 \ + --hash=sha256:da3fe1de14db63a29f0fe274b7a24270b99cfe84186312e18069da0bf9fa4476 + # via + # -r requirements.in + # gremlite + # wsc-grempy-transport +gremlite==0.1.1 \ + --hash=sha256:111b71f0528b41a660d8da66f1367ae39d962094285481a240ba7604f8a8bcd5 \ + --hash=sha256:a0e1173a3170a809a3dc33aa0a82064055552d3f573c4361fa235757cb730982 + # via -r requirements.in h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 @@ -439,7 +722,9 @@ hiredis==2.4.0 \ idna==3.11 \ --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 - # via requests + # via + # requests + # yarl imagesize==1.4.1 \ --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \ --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a @@ -447,7 +732,7 @@ imagesize==1.4.1 \ isodate==0.7.2 \ --hash=sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15 \ --hash=sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6 - # via -r requirements.in + # via gremlinpython itsdangerous==2.2.0 \ --hash=sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef \ --hash=sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173 @@ -683,9 +968,163 @@ mmh3==5.2.0 \ --hash=sha256:fdfd3fb739f4e22746e13ad7ba0c6eedf5f454b18d11249724a388868e308ee4 \ --hash=sha256:ff3d50dc3fe8a98059f99b445dfb62792b5d006c5e0b8f03c6de2813b8376110 # via pottery +multidict==6.7.1 \ + --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ + --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ + --hash=sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581 \ + --hash=sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2 \ + --hash=sha256:08ccb2a6dc72009093ebe7f3f073e5ec5964cba9a706fa94b1a1484039b87941 \ + --hash=sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3 \ + --hash=sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43 \ + --hash=sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962 \ + --hash=sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1 \ + --hash=sha256:0e697826df7eb63418ee190fd06ce9f1803593bb4b9517d08c60d9b9a7f69d8f \ + --hash=sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c \ + --hash=sha256:121a34e5bfa410cdf2c8c49716de160de3b1dbcd86b49656f5681e4543bcd1a8 \ + --hash=sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa \ + --hash=sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6 \ + --hash=sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c \ + --hash=sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991 \ + --hash=sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262 \ + --hash=sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd \ + --hash=sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d \ + --hash=sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d \ + --hash=sha256:1fa6609d0364f4f6f58351b4659a1f3e0e898ba2a8c5cac04cb2c7bc556b0bc5 \ + --hash=sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3 \ + --hash=sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601 \ + --hash=sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505 \ + --hash=sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0 \ + --hash=sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292 \ + --hash=sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed \ + --hash=sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362 \ + --hash=sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511 \ + --hash=sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23 \ + --hash=sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2 \ + --hash=sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb \ + --hash=sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e \ + --hash=sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582 \ + --hash=sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0 \ + --hash=sha256:3943debf0fbb57bdde5901695c11094a9a36723e5c03875f87718ee15ca2f4d2 \ + --hash=sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e \ + --hash=sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d \ + --hash=sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65 \ + --hash=sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a \ + --hash=sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd \ + --hash=sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d \ + --hash=sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108 \ + --hash=sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177 \ + --hash=sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144 \ + --hash=sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5 \ + --hash=sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd \ + --hash=sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5 \ + --hash=sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060 \ + --hash=sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37 \ + --hash=sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56 \ + --hash=sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df \ + --hash=sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963 \ + --hash=sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568 \ + --hash=sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db \ + --hash=sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118 \ + --hash=sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84 \ + --hash=sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f \ + --hash=sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889 \ + --hash=sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71 \ + --hash=sha256:65573858d27cdeaca41893185677dc82395159aa28875a8867af66532d413a8f \ + --hash=sha256:6704fa2b7453b2fb121740555fa1ee20cd98c4d011120caf4d2b8d4e7c76eec0 \ + --hash=sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7 \ + --hash=sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048 \ + --hash=sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8 \ + --hash=sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49 \ + --hash=sha256:6f77ce314a29263e67adadc7e7c1bc699fcb3a305059ab973d038f87caa42ed0 \ + --hash=sha256:749aa54f578f2e5f439538706a475aa844bfa8ef75854b1401e6e528e4937cf9 \ + --hash=sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59 \ + --hash=sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190 \ + --hash=sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709 \ + --hash=sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d \ + --hash=sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c \ + --hash=sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e \ + --hash=sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2 \ + --hash=sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40 \ + --hash=sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3 \ + --hash=sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee \ + --hash=sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609 \ + --hash=sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c \ + --hash=sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445 \ + --hash=sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1 \ + --hash=sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a \ + --hash=sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5 \ + --hash=sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31 \ + --hash=sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8 \ + --hash=sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33 \ + --hash=sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7 \ + --hash=sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca \ + --hash=sha256:98c5787b0a0d9a41d9311eae44c3b76e6753def8d8870ab501320efe75a6a5f8 \ + --hash=sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92 \ + --hash=sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733 \ + --hash=sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429 \ + --hash=sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9 \ + --hash=sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4 \ + --hash=sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6 \ + --hash=sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2 \ + --hash=sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172 \ + --hash=sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981 \ + --hash=sha256:aa23b001d968faef416ff70dc0f1ab045517b9b42a90edd3e9bcdb06479e31d5 \ + --hash=sha256:ac1c665bad8b5d762f5f85ebe4d94130c26965f11de70c708c75671297c776de \ + --hash=sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52 \ + --hash=sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 \ + --hash=sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c \ + --hash=sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2 \ + --hash=sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6 \ + --hash=sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf \ + --hash=sha256:bb08271280173720e9fea9ede98e5231defcbad90f1624bea26f32ec8a956e2f \ + --hash=sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b \ + --hash=sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 \ + --hash=sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a \ + --hash=sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3 \ + --hash=sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b \ + --hash=sha256:c524c6fb8fc342793708ab111c4dbc90ff9abd568de220432500e47e990c0358 \ + --hash=sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6 \ + --hash=sha256:c6b3228e1d80af737b72925ce5fb4daf5a335e49cd7ab77ed7b9fdfbf58c526e \ + --hash=sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1 \ + --hash=sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c \ + --hash=sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5 \ + --hash=sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53 \ + --hash=sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872 \ + --hash=sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e \ + --hash=sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df \ + --hash=sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03 \ + --hash=sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8 \ + --hash=sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a \ + --hash=sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122 \ + --hash=sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a \ + --hash=sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee \ + --hash=sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32 \ + --hash=sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3 \ + --hash=sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489 \ + --hash=sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23 \ + --hash=sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34 \ + --hash=sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75 \ + --hash=sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8 \ + --hash=sha256:eb351f72c26dc9abe338ca7294661aa22969ad8ffe7ef7d5541d19f368dc854a \ + --hash=sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d \ + --hash=sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855 \ + --hash=sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b \ + --hash=sha256:f537b55778cd3cbee430abe3131255d3a78202e0f9ea7ffc6ada893a4bcaeea4 \ + --hash=sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4 \ + --hash=sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d \ + --hash=sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0 \ + --hash=sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba \ + --hash=sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19 + # via + # aiohttp + # yarl neo4j==4.4.13 \ --hash=sha256:d8c74f73c145d54bdb95d2c7f24961a44efedcf840c5f9fe1d1dcb36c669b7a2 # via -r requirements.in +nest-asyncio==1.6.0 \ + --hash=sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe \ + --hash=sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c + # via gremlinpython packaging==26.0 \ --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 @@ -702,6 +1141,132 @@ prettytable==2.5.0 \ --hash=sha256:1411c65d21dca9eaa505ba1d041bed75a6d629ae22f5109a923f4e719cfecba4 \ --hash=sha256:f7da57ba63d55116d65e5acb147bfdfa60dceccabf0d607d6817ee2888a05f2c # via redisgraph +propcache==0.4.1 \ + --hash=sha256:0002004213ee1f36cfb3f9a42b5066100c44276b9b72b4e1504cddd3d692e86e \ + --hash=sha256:0013cb6f8dde4b2a2f66903b8ba740bdfe378c943c4377a200551ceb27f379e4 \ + --hash=sha256:005f08e6a0529984491e37d8dbc3dd86f84bd78a8ceb5fa9a021f4c48d4984be \ + --hash=sha256:031dce78b9dc099f4c29785d9cf5577a3faf9ebf74ecbd3c856a7b92768c3df3 \ + --hash=sha256:05674a162469f31358c30bcaa8883cb7829fa3110bf9c0991fe27d7896c42d85 \ + --hash=sha256:060b16ae65bc098da7f6d25bf359f1f31f688384858204fe5d652979e0015e5b \ + --hash=sha256:120c964da3fdc75e3731aa392527136d4ad35868cc556fd09bb6d09172d9a367 \ + --hash=sha256:15932ab57837c3368b024473a525e25d316d8353016e7cc0e5ba9eb343fbb1cf \ + --hash=sha256:17612831fda0138059cc5546f4d12a2aacfb9e47068c06af35c400ba58ba7393 \ + --hash=sha256:182b51b421f0501952d938dc0b0eb45246a5b5153c50d42b495ad5fb7517c888 \ + --hash=sha256:1cdb7988c4e5ac7f6d175a28a9aa0c94cb6f2ebe52756a3c0cda98d2809a9e37 \ + --hash=sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8 \ + --hash=sha256:1f0978529a418ebd1f49dad413a2b68af33f85d5c5ca5c6ca2a3bed375a7ac60 \ + --hash=sha256:204483131fb222bdaaeeea9f9e6c6ed0cac32731f75dfc1d4a567fc1926477c1 \ + --hash=sha256:296f4c8ed03ca7476813fe666c9ea97869a8d7aec972618671b33a38a5182ef4 \ + --hash=sha256:2ad890caa1d928c7c2965b48f3a3815c853180831d0e5503d35cf00c472f4717 \ + --hash=sha256:2b16ec437a8c8a965ecf95739448dd938b5c7f56e67ea009f4300d8df05f32b7 \ + --hash=sha256:2bb07ffd7eaad486576430c89f9b215f9e4be68c4866a96e97db9e97fead85dc \ + --hash=sha256:333ddb9031d2704a301ee3e506dc46b1fe5f294ec198ed6435ad5b6a085facfe \ + --hash=sha256:357f5bb5c377a82e105e44bd3d52ba22b616f7b9773714bff93573988ef0a5fb \ + --hash=sha256:35c3277624a080cc6ec6f847cbbbb5b49affa3598c4535a0a4682a697aaa5c75 \ + --hash=sha256:364426a62660f3f699949ac8c621aad6977be7126c5807ce48c0aeb8e7333ea6 \ + --hash=sha256:381914df18634f5494334d201e98245c0596067504b9372d8cf93f4bb23e025e \ + --hash=sha256:3d233076ccf9e450c8b3bc6720af226b898ef5d051a2d145f7d765e6e9f9bcff \ + --hash=sha256:3d902a36df4e5989763425a8ab9e98cd8ad5c52c823b34ee7ef307fd50582566 \ + --hash=sha256:3f7124c9d820ba5548d431afb4632301acf965db49e666aa21c305cbe8c6de12 \ + --hash=sha256:405aac25c6394ef275dee4c709be43745d36674b223ba4eb7144bf4d691b7367 \ + --hash=sha256:41a89040cb10bd345b3c1a873b2bf36413d48da1def52f268a055f7398514874 \ + --hash=sha256:43eedf29202c08550aac1d14e0ee619b0430aaef78f85864c1a892294fbc28cf \ + --hash=sha256:473c61b39e1460d386479b9b2f337da492042447c9b685f28be4f74d3529e566 \ + --hash=sha256:49a2dc67c154db2c1463013594c458881a069fcf98940e61a0569016a583020a \ + --hash=sha256:4b536b39c5199b96fc6245eb5fb796c497381d3942f169e44e8e392b29c9ebcc \ + --hash=sha256:4c3c70630930447f9ef1caac7728c8ad1c56bc5015338b20fed0d08ea2480b3a \ + --hash=sha256:4d3df5fa7e36b3225954fba85589da77a0fe6a53e3976de39caf04a0db4c36f1 \ + --hash=sha256:4d7af63f9f93fe593afbf104c21b3b15868efb2c21d07d8732c0c4287e66b6a6 \ + --hash=sha256:501d20b891688eb8e7aa903021f0b72d5a55db40ffaab27edefd1027caaafa61 \ + --hash=sha256:521a463429ef54143092c11a77e04056dd00636f72e8c45b70aaa3140d639726 \ + --hash=sha256:5558992a00dfd54ccbc64a32726a3357ec93825a418a401f5cc67df0ac5d9e49 \ + --hash=sha256:55c72fd6ea2da4c318e74ffdf93c4fe4e926051133657459131a95c846d16d44 \ + --hash=sha256:564d9f0d4d9509e1a870c920a89b2fec951b44bf5ba7d537a9e7c1ccec2c18af \ + --hash=sha256:580e97762b950f993ae618e167e7be9256b8353c2dcd8b99ec100eb50f5286aa \ + --hash=sha256:5a103c3eb905fcea0ab98be99c3a9a5ab2de60228aa5aceedc614c0281cf6153 \ + --hash=sha256:5c3310452e0d31390da9035c348633b43d7e7feb2e37be252be6da45abd1abcc \ + --hash=sha256:5d4e2366a9c7b837555cf02fb9be2e3167d333aff716332ef1b7c3a142ec40c5 \ + --hash=sha256:5fd37c406dd6dc85aa743e214cef35dc54bbdd1419baac4f6ae5e5b1a2976938 \ + --hash=sha256:60a8fda9644b7dfd5dece8c61d8a85e271cb958075bfc4e01083c148b61a7caf \ + --hash=sha256:66c1f011f45a3b33d7bcb22daed4b29c0c9e2224758b6be00686731e1b46f925 \ + --hash=sha256:671538c2262dadb5ba6395e26c1731e1d52534bfe9ae56d0b5573ce539266aa8 \ + --hash=sha256:678ae89ebc632c5c204c794f8dab2837c5f159aeb59e6ed0539500400577298c \ + --hash=sha256:67fad6162281e80e882fb3ec355398cf72864a54069d060321f6cd0ade95fe85 \ + --hash=sha256:6918ecbd897443087a3b7cd978d56546a812517dcaaca51b49526720571fa93e \ + --hash=sha256:6f6ff873ed40292cd4969ef5310179afd5db59fdf055897e282485043fc80ad0 \ + --hash=sha256:6f8b465489f927b0df505cbe26ffbeed4d6d8a2bbc61ce90eb074ff129ef0ab1 \ + --hash=sha256:71b749281b816793678ae7f3d0d84bd36e694953822eaad408d682efc5ca18e0 \ + --hash=sha256:74c1fb26515153e482e00177a1ad654721bf9207da8a494a0c05e797ad27b992 \ + --hash=sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db \ + --hash=sha256:824e908bce90fb2743bd6b59db36eb4f45cd350a39637c9f73b1c1ea66f5b75f \ + --hash=sha256:8326e144341460402713f91df60ade3c999d601e7eb5ff8f6f7862d54de0610d \ + --hash=sha256:8873eb4460fd55333ea49b7d189749ecf6e55bf85080f11b1c4530ed3034cba1 \ + --hash=sha256:89eb3fa9524f7bec9de6e83cf3faed9d79bffa560672c118a96a171a6f55831e \ + --hash=sha256:8c9b3cbe4584636d72ff556d9036e0c9317fa27b3ac1f0f558e7e84d1c9c5900 \ + --hash=sha256:8e57061305815dfc910a3634dcf584f08168a8836e6999983569f51a8544cd89 \ + --hash=sha256:929d7cbe1f01bb7baffb33dc14eb5691c95831450a26354cd210a8155170c93a \ + --hash=sha256:92d1935ee1f8d7442da9c0c4fa7ac20d07e94064184811b685f5c4fada64553b \ + --hash=sha256:948dab269721ae9a87fd16c514a0a2c2a1bdb23a9a61b969b0f9d9ee2968546f \ + --hash=sha256:981333cb2f4c1896a12f4ab92a9cc8f09ea664e9b7dbdc4eff74627af3a11c0f \ + --hash=sha256:990f6b3e2a27d683cb7602ed6c86f15ee6b43b1194736f9baaeb93d0016633b1 \ + --hash=sha256:99d43339c83aaf4d32bda60928231848eee470c6bda8d02599cc4cebe872d183 \ + --hash=sha256:9a0bd56e5b100aef69bd8562b74b46254e7c8812918d3baa700c8a8009b0af66 \ + --hash=sha256:9a52009f2adffe195d0b605c25ec929d26b36ef986ba85244891dee3b294df21 \ + --hash=sha256:9d2b6caef873b4f09e26ea7e33d65f42b944837563a47a94719cc3544319a0db \ + --hash=sha256:9f302f4783709a78240ebc311b793f123328716a60911d667e0c036bc5dcbded \ + --hash=sha256:a0ee98db9c5f80785b266eb805016e36058ac72c51a064040f2bc43b61101cdb \ + --hash=sha256:a129e76735bc792794d5177069691c3217898b9f5cee2b2661471e52ffe13f19 \ + --hash=sha256:a78372c932c90ee474559c5ddfffd718238e8673c340dc21fe45c5b8b54559a0 \ + --hash=sha256:a9695397f85973bb40427dedddf70d8dc4a44b22f1650dd4af9eedf443d45165 \ + --hash=sha256:ab08df6c9a035bee56e31af99be621526bd237bea9f32def431c656b29e41778 \ + --hash=sha256:ab2943be7c652f09638800905ee1bab2c544e537edb57d527997a24c13dc1455 \ + --hash=sha256:ab4c29b49d560fe48b696cdcb127dd36e0bc2472548f3bf56cc5cb3da2b2984f \ + --hash=sha256:af223b406d6d000830c6f65f1e6431783fc3f713ba3e6cc8c024d5ee96170a4b \ + --hash=sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237 \ + --hash=sha256:bcc9aaa5d80322bc2fb24bb7accb4a30f81e90ab8d6ba187aec0744bc302ad81 \ + --hash=sha256:c07fda85708bc48578467e85099645167a955ba093be0a2dcba962195676e859 \ + --hash=sha256:c0d4b719b7da33599dfe3b22d3db1ef789210a0597bc650b7cee9c77c2be8c5c \ + --hash=sha256:c0ef0aaafc66fbd87842a3fe3902fd889825646bc21149eafe47be6072725835 \ + --hash=sha256:c2b5e7db5328427c57c8e8831abda175421b709672f6cfc3d630c3b7e2146393 \ + --hash=sha256:c30b53e7e6bda1d547cabb47c825f3843a0a1a42b0496087bb58d8fedf9f41b5 \ + --hash=sha256:c80ee5802e3fb9ea37938e7eecc307fb984837091d5fd262bb37238b1ae97641 \ + --hash=sha256:c9b822a577f560fbd9554812526831712c1436d2c046cedee4c3796d3543b144 \ + --hash=sha256:cae65ad55793da34db5f54e4029b89d3b9b9490d8abe1b4c7ab5d4b8ec7ebf74 \ + --hash=sha256:cb2d222e72399fcf5890d1d5cc1060857b9b236adff2792ff48ca2dfd46c81db \ + --hash=sha256:cbc3b6dfc728105b2a57c06791eb07a94229202ea75c59db644d7d496b698cac \ + --hash=sha256:cd547953428f7abb73c5ad82cbb32109566204260d98e41e5dfdc682eb7f8403 \ + --hash=sha256:cfc27c945f422e8b5071b6e93169679e4eb5bf73bbcbf1ba3ae3a83d2f78ebd9 \ + --hash=sha256:d472aeb4fbf9865e0c6d622d7f4d54a4e101a89715d8904282bb5f9a2f476c3f \ + --hash=sha256:d62cdfcfd89ccb8de04e0eda998535c406bf5e060ffd56be6c586cbcc05b3311 \ + --hash=sha256:d82ad62b19645419fe79dd63b3f9253e15b30e955c0170e5cebc350c1844e581 \ + --hash=sha256:d8f353eb14ee3441ee844ade4277d560cdd68288838673273b978e3d6d2c8f36 \ + --hash=sha256:daede9cd44e0f8bdd9e6cc9a607fc81feb80fae7a5fc6cecaff0e0bb32e42d00 \ + --hash=sha256:db65d2af507bbfbdcedb254a11149f894169d90488dd3e7190f7cdcb2d6cd57a \ + --hash=sha256:dee69d7015dc235f526fe80a9c90d65eb0039103fe565776250881731f06349f \ + --hash=sha256:e153e9cd40cc8945138822807139367f256f89c6810c2634a4f6902b52d3b4e2 \ + --hash=sha256:e35b88984e7fa64aacecea39236cee32dd9bd8c55f57ba8a75cf2399553f9bd7 \ + --hash=sha256:e53f3a38d3510c11953f3e6a33f205c6d1b001129f972805ca9b42fc308bc239 \ + --hash=sha256:e9b0d8d0845bbc4cfcdcbcdbf5086886bc8157aa963c31c777ceff7846c77757 \ + --hash=sha256:ec17c65562a827bba85e3872ead335f95405ea1674860d96483a02f5c698fa72 \ + --hash=sha256:ecef2343af4cc68e05131e45024ba34f6095821988a9d0a02aa7c73fcc448aa9 \ + --hash=sha256:ed5a841e8bb29a55fb8159ed526b26adc5bdd7e8bd7bf793ce647cb08656cdf4 \ + --hash=sha256:ee17f18d2498f2673e432faaa71698032b0127ebf23ae5974eeaf806c279df24 \ + --hash=sha256:f048da1b4f243fc44f205dfd320933a951b8d89e0afd4c7cacc762a8b9165207 \ + --hash=sha256:f10207adf04d08bec185bae14d9606a1444715bc99180f9331c9c02093e1959e \ + --hash=sha256:f1d2f90aeec838a52f1c1a32fe9a619fefd5e411721a9117fbf82aea638fe8a1 \ + --hash=sha256:f48107a8c637e80362555f37ecf49abe20370e557cc4ab374f04ec4423c97c3d \ + --hash=sha256:f7ee0e597f495cf415bcbd3da3caa3bd7e816b74d0d52b8145954c5e6fd3ff37 \ + --hash=sha256:f93243fdc5657247533273ac4f86ae106cc6445a0efacb9a1bfe982fcfefd90c \ + --hash=sha256:f95393b4d66bfae908c3ca8d169d5f79cd65636ae15b5e7a4f6e67af675adb0e \ + --hash=sha256:fc38cba02d1acba4e2869eef1a57a43dfbd3d49a59bf90dda7444ec2be6a5570 \ + --hash=sha256:fd0858c20f078a32cf55f7e81473d96dcf3b93fd2ccdb3d40fdf54b8573df3af \ + --hash=sha256:fd138803047fb4c062b1c1dd95462f5209456bfab55c734458f15d11da288f8f \ + --hash=sha256:fd2dbc472da1f772a4dae4fa24be938a6c544671a912e30529984dd80400cd88 \ + --hash=sha256:fd6f30fdcf9ae2a70abd34da54f18da086160e4d7d9251f81f3da0ff84fc5a48 \ + --hash=sha256:fe49d0a85038f36ba9e3ffafa1103e61170b28e95b16622e11be0a0ea07c6781 + # via + # aiohttp + # yarl pycparser==3.0 \ --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 @@ -851,6 +1416,7 @@ typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via + # aiosignal # beautifulsoup4 # pottery urllib3==2.6.3 \ @@ -865,17 +1431,153 @@ wcwidth==0.6.0 \ --hash=sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad \ --hash=sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159 # via prettytable -websocket-client==1.3.2 \ - --hash=sha256:50b21db0058f7a953d67cc0445be4b948d7fc196ecbeb8083d68d94628e4abf6 \ - --hash=sha256:722b171be00f2b90e1d4fb2f2b53146a536ca38db1da8ff49c972a4e1365d0ef - # via -r requirements.in +websocket-client==1.9.0 \ + --hash=sha256:9e813624b6eb619999a97dc7958469217c3176312b3a16a4bd1bc7e08a46ec98 \ + --hash=sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef + # via wsc-grempy-transport werkzeug==3.1.5 \ --hash=sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc \ --hash=sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67 # via # flask # flask-login +wsc-grempy-transport==0.1.0 \ + --hash=sha256:6c24d1c0d164057935e5fe1dc8a74bc8c0cf937dac2cd5528d612b00d416a0b1 \ + --hash=sha256:be32e2e98973c25392498a61f519ec49d581bdaa9e448abd1b2f975d30f0ad58 + # via -r requirements.in wsproto==1.3.2 \ --hash=sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584 \ --hash=sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294 # via simple-websocket +yarl==1.22.0 \ + --hash=sha256:01e73b85a5434f89fc4fe27dcda2aff08ddf35e4d47bbbea3bdcd25321af538a \ + --hash=sha256:029866bde8d7b0878b9c160e72305bbf0a7342bcd20b9999381704ae03308dc8 \ + --hash=sha256:078278b9b0b11568937d9509b589ee83ef98ed6d561dfe2020e24a9fd08eaa2b \ + --hash=sha256:078a8aefd263f4d4f923a9677b942b445a2be970ca24548a8102689a3a8ab8da \ + --hash=sha256:07a524d84df0c10f41e3ee918846e1974aba4ec017f990dc735aad487a0bdfdf \ + --hash=sha256:088e4e08f033db4be2ccd1f34cf29fe994772fb54cfe004bbf54db320af56890 \ + --hash=sha256:0b5bcc1a9c4839e7e30b7b30dd47fe5e7e44fb7054ec29b5bb8d526aa1041093 \ + --hash=sha256:0cf71bf877efeac18b38d3930594c0948c82b64547c1cf420ba48722fe5509f6 \ + --hash=sha256:0d6e6885777af0f110b0e5d7e5dda8b704efed3894da26220b7f3d887b839a79 \ + --hash=sha256:0dd9a702591ca2e543631c2a017e4a547e38a5c0f29eece37d9097e04a7ac683 \ + --hash=sha256:10619d9fdee46d20edc49d3479e2f8269d0779f1b031e6f7c2aa1c76be04b7ed \ + --hash=sha256:131a085a53bfe839a477c0845acf21efc77457ba2bcf5899618136d64f3303a2 \ + --hash=sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff \ + --hash=sha256:139718f35149ff544caba20fce6e8a2f71f1e39b92c700d8438a0b1d2a631a02 \ + --hash=sha256:14291620375b1060613f4aab9ebf21850058b6b1b438f386cc814813d901c60b \ + --hash=sha256:1834bb90991cc2999f10f97f5f01317f99b143284766d197e43cd5b45eb18d03 \ + --hash=sha256:1ab72135b1f2db3fed3997d7e7dc1b80573c67138023852b6efb336a5eae6511 \ + --hash=sha256:1e7ce67c34138a058fd092f67d07a72b8e31ff0c9236e751957465a24b28910c \ + --hash=sha256:1e8fbaa7cec507aa24ea27a01456e8dd4b6fab829059b69844bd348f2d467124 \ + --hash=sha256:22965c2af250d20c873cdbee8ff958fb809940aeb2e74ba5f20aaf6b7ac8c70c \ + --hash=sha256:22b029f2881599e2f1b06f8f1db2ee63bd309e2293ba2d566e008ba12778b8da \ + --hash=sha256:243dda95d901c733f5b59214d28b0120893d91777cb8aa043e6ef059d3cddfe2 \ + --hash=sha256:2ca6fd72a8cd803be290d42f2dec5cdcd5299eeb93c2d929bf060ad9efaf5de0 \ + --hash=sha256:2e4e1f6f0b4da23e61188676e3ed027ef0baa833a2e633c29ff8530800edccba \ + --hash=sha256:31f0b53913220599446872d757257be5898019c85e7971599065bc55065dc99d \ + --hash=sha256:334b8721303e61b00019474cc103bdac3d7b1f65e91f0bfedeec2d56dfe74b53 \ + --hash=sha256:33e32a0dd0c8205efa8e83d04fc9f19313772b78522d1bdc7d9aed706bfd6138 \ + --hash=sha256:34b36c2c57124530884d89d50ed2c1478697ad7473efd59cfd479945c95650e4 \ + --hash=sha256:3aa27acb6de7a23785d81557577491f6c38a5209a254d1191519d07d8fe51748 \ + --hash=sha256:3b06bcadaac49c70f4c88af4ffcfbe3dc155aab3163e75777818092478bcbbe7 \ + --hash=sha256:3b7c88eeef021579d600e50363e0b6ee4f7f6f728cd3486b9d0f3ee7b946398d \ + --hash=sha256:3e2daa88dc91870215961e96a039ec73e4937da13cf77ce17f9cad0c18df3503 \ + --hash=sha256:3ea66b1c11c9150f1372f69afb6b8116f2dd7286f38e14ea71a44eee9ec51b9d \ + --hash=sha256:42188e6a615c1a75bcaa6e150c3fe8f3e8680471a6b10150c5f7e83f47cc34d2 \ + --hash=sha256:433885ab5431bc3d3d4f2f9bd15bfa1614c522b0f1405d62c4f926ccd69d04fa \ + --hash=sha256:437840083abe022c978470b942ff832c3940b2ad3734d424b7eaffcd07f76737 \ + --hash=sha256:4398557cbf484207df000309235979c79c4356518fd5c99158c7d38203c4da4f \ + --hash=sha256:45c2842ff0e0d1b35a6bf1cd6c690939dacb617a70827f715232b2e0494d55d1 \ + --hash=sha256:47743b82b76d89a1d20b83e60d5c20314cbd5ba2befc9cda8f28300c4a08ed4d \ + --hash=sha256:4792b262d585ff0dff6bcb787f8492e40698443ec982a3568c2096433660c694 \ + --hash=sha256:47d8a5c446df1c4db9d21b49619ffdba90e77c89ec6e283f453856c74b50b9e3 \ + --hash=sha256:47fdb18187e2a4e18fda2c25c05d8251a9e4a521edaed757fef033e7d8498d9a \ + --hash=sha256:4c52a6e78aef5cf47a98ef8e934755abf53953379b7d53e68b15ff4420e6683d \ + --hash=sha256:4dcc74149ccc8bba31ce1944acee24813e93cfdee2acda3c172df844948ddf7b \ + --hash=sha256:50678a3b71c751d58d7908edc96d332af328839eea883bb554a43f539101277a \ + --hash=sha256:51af598701f5299012b8416486b40fceef8c26fc87dc6d7d1f6fc30609ea0aa6 \ + --hash=sha256:594fcab1032e2d2cc3321bb2e51271e7cd2b516c7d9aee780ece81b07ff8244b \ + --hash=sha256:595697f68bd1f0c1c159fcb97b661fc9c3f5db46498043555d04805430e79bea \ + --hash=sha256:59c189e3e99a59cf8d83cbb31d4db02d66cda5a1a4374e8a012b51255341abf5 \ + --hash=sha256:5a3bf7f62a289fa90f1990422dc8dff5a458469ea71d1624585ec3a4c8d6960f \ + --hash=sha256:5c401e05ad47a75869c3ab3e35137f8468b846770587e70d71e11de797d113df \ + --hash=sha256:5cdac20da754f3a723cceea5b3448e1a2074866406adeb4ef35b469d089adb8f \ + --hash=sha256:5d0fcda9608875f7d052eff120c7a5da474a6796fe4d83e152e0e4d42f6d1a9b \ + --hash=sha256:5dbeefd6ca588b33576a01b0ad58aa934bc1b41ef89dee505bf2932b22ddffba \ + --hash=sha256:62441e55958977b8167b2709c164c91a6363e25da322d87ae6dd9c6019ceecf9 \ + --hash=sha256:663e1cadaddae26be034a6ab6072449a8426ddb03d500f43daf952b74553bba0 \ + --hash=sha256:669930400e375570189492dc8d8341301578e8493aec04aebc20d4717f899dd6 \ + --hash=sha256:68986a61557d37bb90d3051a45b91fa3d5c516d177dfc6dd6f2f436a07ff2b6b \ + --hash=sha256:6944b2dc72c4d7f7052683487e3677456050ff77fcf5e6204e98caf785ad1967 \ + --hash=sha256:6a635ea45ba4ea8238463b4f7d0e721bad669f80878b7bfd1f89266e2ae63da2 \ + --hash=sha256:6c5010a52015e7c70f86eb967db0f37f3c8bd503a695a49f8d45700144667708 \ + --hash=sha256:6dcbb0829c671f305be48a7227918cfcd11276c2d637a8033a99a02b67bf9eda \ + --hash=sha256:70dfd4f241c04bd9239d53b17f11e6ab672b9f1420364af63e8531198e3f5fe8 \ + --hash=sha256:719ae08b6972befcba4310e49edb1161a88cdd331e3a694b84466bd938a6ab10 \ + --hash=sha256:75976c6945d85dbb9ee6308cd7ff7b1fb9409380c82d6119bd778d8fcfe2931c \ + --hash=sha256:7861058d0582b847bc4e3a4a4c46828a410bca738673f35a29ba3ca5db0b473b \ + --hash=sha256:792a2af6d58177ef7c19cbf0097aba92ca1b9cb3ffdd9c7470e156c8f9b5e028 \ + --hash=sha256:8009b3173bcd637be650922ac455946197d858b3630b6d8787aa9e5c4564533e \ + --hash=sha256:80ddf7a5f8c86cb3eb4bc9028b07bbbf1f08a96c5c0bc1244be5e8fefcb94147 \ + --hash=sha256:8218f4e98d3c10d683584cb40f0424f4b9fd6e95610232dd75e13743b070ee33 \ + --hash=sha256:84fc3ec96fce86ce5aa305eb4aa9358279d1aa644b71fab7b8ed33fe3ba1a7ca \ + --hash=sha256:852863707010316c973162e703bddabec35e8757e67fcb8ad58829de1ebc8590 \ + --hash=sha256:8884d8b332a5e9b88e23f60bb166890009429391864c685e17bd73a9eda9105c \ + --hash=sha256:8dee9c25c74997f6a750cd317b8ca63545169c098faee42c84aa5e506c819b53 \ + --hash=sha256:939fe60db294c786f6b7c2d2e121576628468f65453d86b0fe36cb52f987bd74 \ + --hash=sha256:99b6fc1d55782461b78221e95fc357b47ad98b041e8e20f47c1411d0aacddc60 \ + --hash=sha256:9d7672ecf7557476642c88497c2f8d8542f8e36596e928e9bcba0e42e1e7d71f \ + --hash=sha256:9f6d73c1436b934e3f01df1e1b21ff765cd1d28c77dfb9ace207f746d4610ee1 \ + --hash=sha256:9fb17ea16e972c63d25d4a97f016d235c78dd2344820eb35bc034bc32012ee27 \ + --hash=sha256:a49370e8f711daec68d09b821a34e1167792ee2d24d405cbc2387be4f158b520 \ + --hash=sha256:a4fcfc8eb2c34148c118dfa02e6427ca278bfd0f3df7c5f99e33d2c0e81eae3e \ + --hash=sha256:a899cbd98dce6f5d8de1aad31cb712ec0a530abc0a86bd6edaa47c1090138467 \ + --hash=sha256:a9b1ba5610a4e20f655258d5a1fdc7ebe3d837bb0e45b581398b99eb98b1f5ca \ + --hash=sha256:af74f05666a5e531289cb1cc9c883d1de2088b8e5b4de48004e5ca8a830ac859 \ + --hash=sha256:b0748275abb8c1e1e09301ee3cf90c8a99678a4e92e4373705f2a2570d581273 \ + --hash=sha256:b266bd01fedeffeeac01a79ae181719ff848a5a13ce10075adbefc8f1daee70e \ + --hash=sha256:b4f15793aa49793ec8d1c708ab7f9eded1aa72edc5174cae703651555ed1b601 \ + --hash=sha256:b580e71cac3f8113d3135888770903eaf2f507e9421e5697d6ee6d8cd1c7f054 \ + --hash=sha256:b6a6f620cfe13ccec221fa312139135166e47ae169f8253f72a0abc0dae94376 \ + --hash=sha256:b790b39c7e9a4192dc2e201a282109ed2985a1ddbd5ac08dc56d0e121400a8f7 \ + --hash=sha256:b85b982afde6df99ecc996990d4ad7ccbdbb70e2a4ba4de0aecde5922ba98a0b \ + --hash=sha256:b8a0588521a26bf92a57a1705b77b8b59044cdceccac7151bd8d229e66b8dedb \ + --hash=sha256:ba440ae430c00eee41509353628600212112cd5018d5def7e9b05ea7ac34eb65 \ + --hash=sha256:bca03b91c323036913993ff5c738d0842fc9c60c4648e5c8d98331526df89784 \ + --hash=sha256:bebf8557577d4401ba8bd9ff33906f1376c877aa78d1fe216ad01b4d6745af71 \ + --hash=sha256:bec03d0d388060058f5d291a813f21c011041938a441c593374da6077fe21b1b \ + --hash=sha256:bf4a21e58b9cde0e401e683ebd00f6ed30a06d14e93f7c8fd059f8b6e8f87b6a \ + --hash=sha256:c0232bce2170103ec23c454e54a57008a9a72b5d1c3105dc2496750da8cfa47c \ + --hash=sha256:c4647674b6150d2cae088fc07de2738a84b8bcedebef29802cf0b0a82ab6face \ + --hash=sha256:c7044802eec4524fde550afc28edda0dd5784c4c45f0be151a2d3ba017daca7d \ + --hash=sha256:c7bd6683587567e5a49ee6e336e0612bec8329be1b7d4c8af5687dcdeb67ee1e \ + --hash=sha256:ca1f59c4e1ab6e72f0a23c13fca5430f889634166be85dbf1013683e49e3278e \ + --hash=sha256:cb95a9b1adaa48e41815a55ae740cfda005758104049a640a398120bf02515ca \ + --hash=sha256:cfebc0ac8333520d2d0423cbbe43ae43c8838862ddb898f5ca68565e395516e9 \ + --hash=sha256:d332fc2e3c94dad927f2112395772a4e4fedbcf8f80efc21ed7cdfae4d574fdb \ + --hash=sha256:d3e32536234a95f513bd374e93d717cf6b2231a791758de6c509e3653f234c95 \ + --hash=sha256:d5372ca1df0f91a86b047d1277c2aaf1edb32d78bbcefffc81b40ffd18f027ed \ + --hash=sha256:d77e1b2c6d04711478cb1c4ab90db07f1609ccf06a287d5607fcd90dc9863acf \ + --hash=sha256:d947071e6ebcf2e2bee8fce76e10faca8f7a14808ca36a910263acaacef08eca \ + --hash=sha256:dd7afd3f8b0bfb4e0d9fc3c31bfe8a4ec7debe124cfd90619305def3c8ca8cd2 \ + --hash=sha256:de6b9a04c606978fdfe72666fa216ffcf2d1a9f6a381058d4378f8d7b1e5de62 \ + --hash=sha256:e1651bf8e0398574646744c1885a41198eba53dc8a9312b954073f845c90a8df \ + --hash=sha256:e1b329cb8146d7b736677a2440e422eadd775d1806a81db2d4cded80a48efc1a \ + --hash=sha256:e1b51bebd221006d3d2f95fbe124b22b247136647ae5dcc8c7acafba66e5ee67 \ + --hash=sha256:e340382d1afa5d32b892b3ff062436d592ec3d692aeea3bef3a5cfe11bbf8c6f \ + --hash=sha256:e4b582bab49ac33c8deb97e058cd67c2c50dac0dd134874106d9c774fd272529 \ + --hash=sha256:e51ac5435758ba97ad69617e13233da53908beccc6cfcd6c34bbed8dcbede486 \ + --hash=sha256:e5542339dcf2747135c5c85f68680353d5cb9ffd741c0f2e8d832d054d41f35a \ + --hash=sha256:e6438cc8f23a9c1478633d216b16104a586b9761db62bfacb6425bac0a36679e \ + --hash=sha256:e81fda2fb4a07eda1a2252b216aa0df23ebcd4d584894e9612e80999a78fd95b \ + --hash=sha256:ea70f61a47f3cc93bdf8b2f368ed359ef02a01ca6393916bc8ff877427181e74 \ + --hash=sha256:ebd4549b108d732dba1d4ace67614b9545b21ece30937a63a65dd34efa19732d \ + --hash=sha256:efb07073be061c8f79d03d04139a80ba33cbd390ca8f0297aae9cce6411e4c6b \ + --hash=sha256:f0d97c18dfd9a9af4490631905a3f131a8e4c9e80a39353919e2cfed8f00aedc \ + --hash=sha256:f1e09112a2c31ffe8d80be1b0988fa6a18c5d5cad92a9ffbb1c04c91bfe52ad2 \ + --hash=sha256:f3d7a87a78d46a2e3d5b72587ac14b4c16952dd0887dbb051451eceac774411e \ + --hash=sha256:f4afb5c34f2c6fecdcc182dfcfc6af6cccf1aa923eed4d6a12e9d96904e1a0d8 \ + --hash=sha256:f6d2cb59377d99718913ad9a151030d6f83ef420a2b8f521d94609ecc106ee82 \ + --hash=sha256:f87ac53513d22240c7d59203f25cc3beac1e574c6cd681bbfd321987b69f95fd \ + --hash=sha256:ff86011bd159a9d2dfc89c34cfd8aff12875980e3bd6a39ff097887520e60249 + # via aiohttp diff --git a/server/req/test-requirements.txt b/server/req/test-requirements.txt index 0d840e9..9c9a390 100644 --- a/server/req/test-requirements.txt +++ b/server/req/test-requirements.txt @@ -2,18 +2,14 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --generate-hashes test-requirements.in +# pip-compile --generate-hashes --no-annotate test-requirements.in # attrs==25.4.0 \ --hash=sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11 \ --hash=sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373 - # via pytest beautifulsoup4==4.14.3 \ --hash=sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb \ --hash=sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86 - # via - # -c requirements.txt - # -r test-requirements.in coverage[toml]==7.13.4 \ --hash=sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246 \ --hash=sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459 \ @@ -121,33 +117,21 @@ coverage[toml]==7.13.4 \ --hash=sha256:fb07dc5da7e849e2ad31a5d74e9bece81f30ecf5a42909d0a695f8bd1874d6af \ --hash=sha256:fb26a934946a6afe0e326aebe0730cdff393a8bc0bbb65a2f41e30feddca399c \ --hash=sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0 - # via pytest-cov displaylang==0.24.1 \ --hash=sha256:5c64a3272d9476aa7259f36b1505ac8ffb90667dacf46bdd119ec5433805ef75 \ --hash=sha256:e2876d95819c6b55fa1cd4b0cb128e0eedd9aafd75a5eec159228f43eb398894 - # via - # displaylang-sympy - # pfsc-examp displaylang-sympy==0.10.4 \ --hash=sha256:6192b75d8b4e223530397cf1a3d79bad91bf414503b6275b9d86ccfaec5a2c53 \ --hash=sha256:b9cfb712588baf226abab2dc82bc3bcdf30e903a98155793f5f6f91d87b823cc - # via pfsc-examp iniconfig==2.3.0 \ --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 - # via pytest jinja2==3.1.6 \ --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 - # via - # -c requirements.txt - # pfsc-examp lark067==0.6.7 \ --hash=sha256:3c482e6f33c173e884afa44cdd411b8af02c043dec2588511a4041f7edafc662 \ --hash=sha256:a4c0c1ca147546b56fbe544a0ca38f689c9de41d2f624b7988c09a943ee48ed3 - # via - # -c requirements.txt - # pfsc-examp markupsafe==3.0.3 \ --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ @@ -238,58 +222,33 @@ markupsafe==3.0.3 \ --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 - # via - # -c requirements.txt - # jinja2 - # pfsc-examp mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c - # via displaylang-sympy packaging==26.0 \ --hash=sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4 \ --hash=sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 - # via - # -c requirements.txt - # pytest pfsc-examp==0.25.0 \ --hash=sha256:0a4f3f6c488541192dee96fd6eabb8dc38d863f100eaded8aa4e868a368e81db \ --hash=sha256:9d3cf145eb55133e8dafba31d345ad80561b1c0dfcef88b8207aa5dd6fc5d4ac - # via -r test-requirements.in pfsc-util==0.22.8 \ --hash=sha256:2967e36233fc05b303257d1639317e9d8db3e50f8054cc5f48b8da6dd36042a0 \ --hash=sha256:7f6c6e2e101cb76e455a4d1cf31faeb35feca44151949773e41184b30092f268 - # via - # -c requirements.txt - # displaylang - # pfsc-examp pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 - # via pytest pytest==7.2.0 \ --hash=sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71 \ --hash=sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59 - # via - # -r test-requirements.in - # pytest-cov pytest-cov==4.0.0 \ --hash=sha256:2feb1b751d66a8bd934e5edfa2e961d11309dc37b73b0eabe73b5945fee20f6b \ --hash=sha256:996b79efde6433cdbd0088872dbc5fb3ed7fe1578b68cdbba634f14bb8dd0470 - # via -r test-requirements.in soupsieve==2.8.3 \ --hash=sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349 \ --hash=sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95 - # via - # -c requirements.txt - # beautifulsoup4 typeguard==2.13.3 \ --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \ --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1 - # via displaylang typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 - # via - # -c requirements.txt - # beautifulsoup4 From 239a0da9b51673a1e7c26b46d4c19346516e0921 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Sun, 17 Nov 2024 16:00:38 -0500 Subject: [PATCH 10/35] Review and improve queries in gremlin/reader * One method, `check_approvals_under_anno()` actually gets an improved runtime (saving about 44% over old times). * We build some experimental apparatus for trying to make the `get_existing_objects()` method more efficient, but ultimately just stick with the existing method. * Added a bunch of comments to clarify why certain queries are written the way they are. --- server/pfsc/gdb/gremlin/reader.py | 206 +++++++++++++++--- .../resources/repo/moo/bar/v0.3.4/__.pfsc | 0 server/tests/test_cf.py | 5 + server/tests/util/__init__.py | 15 ++ 4 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 server/tests/resources/repo/moo/bar/v0.3.4/__.pfsc diff --git a/server/pfsc/gdb/gremlin/reader.py b/server/pfsc/gdb/gremlin/reader.py index 381a77a..bd34f00 100644 --- a/server/pfsc/gdb/gremlin/reader.py +++ b/server/pfsc/gdb/gremlin/reader.py @@ -15,9 +15,10 @@ # --------------------------------------------------------------------------- # import json +import time from gremlin_python.process.graph_traversal import __ -from gremlin_python.process.traversal import TextP +from gremlin_python.process.traversal import TextP, P from pfsc.constants import IndexType, WIP_TAG from pfsc.gdb.k import make_kNode_from_jNode, make_kReln_from_jReln @@ -69,6 +70,15 @@ def find_move_conjugate(self, libpath, major): # The node in question could not be found. return None + # Note: the usages here of `.out_e().in_v()` are deliberate, and can *not* + # be simplified to `out()`. The reason for this is that we need both edges and + # vertices to be reported by the `path()` step. + # + # Note 2: Because the `repeat()` step is following outgoing UNDER edges, there + # is no need for a `limit(1)`. No vertex in our database can ever have more than + # one outgoing `UNDER` edge, so the BFS conducted by `repeat()` has nowhere else + # to go, no other paths to explore. Once we hit the `until()` condition, the search + # will stop, without need of a `limit(1)` to make it stop. tr2 = self.g.V(tr1.next()) \ .until(__.out_e(IndexType.MOVE)) \ .repeat(__.out_e(IndexType.UNDER).in_v()) \ @@ -77,8 +87,8 @@ def find_move_conjugate(self, libpath, major): __.path() \ .by(__.id_()) \ # The UNDER relns have a 'segment' property, but the MOVE - # relns do not, so we need a coalesce. (On Neptune the entire - # path query fails if any part fails its `by()`.) + # relns do not, so we need a coalesce. (A path step returns no + # result at all if any of its `by()` modulators is non-productive.) .by(__.coalesce( __.values('segment'), __.constant(0) @@ -107,7 +117,8 @@ def find_move_conjugate(self, libpath, major): props = self.g.V(sp).element_map().next() return make_kNode_from_jNode(props) - segments = [p[n - 4 - 2 * i] for i in range((n - 3) // 2)] + # `Path` instances do not accept slices, so we slice the `objects` attr, which is a list. + segments = p.objects[-4::-2] tr = self.g.V(sp) for seg in segments: @@ -115,8 +126,105 @@ def find_move_conjugate(self, libpath, major): props = tr.element_map().next() return make_kNode_from_jNode(props) + ################################################################################ + # Get existing objects + + """ + What is the most efficient way to query the database for all existing objects (both + vertices and edges) at and under a given modpath, and for a given major version? + + At this time we have written two different methods, which are implemented in the + `get_existing_j_objects_0()` and `get_existing_j_objects_1()` methods below. + + For now, we are simply using method 0, but I'm not convinced we really know which + method (one of these, or perhaps another we haven't thought of yet) is really best. + + If you want to experiment with these methods again, here is how to do it: + + * In `get_existing_objects()`, pass `[0, 1]` to the `methods` kwarg of + the `get_existing_objects_MULTI()` method. The latter tries both methods + and compares their results. + + * In your `instance/.env`, set + + DUPLICATE_TEST_HIST_LIT=1 + + which will cause there to be a `v1.0.0` of the `test.hist.lit` repo. This is + important so that we can see, for a fairly large repo, how long it takes to + fetch all existing objects for the previous version. + See also the `tests.util.gather_repo_info()` function on this. + """ + def get_existing_objects(self, modpath, major, recursive): - # Nodes + # For now we're sticking with method 0. + # Tests showed that both methods were taking about the same time. + # I do wonder which one might scale better, but for now we leave this question for future work. + methods = [0] + # methods = [0, 1] + return self.get_existing_objects_MULTI(modpath, major, recursive, methods=methods) + + def get_existing_objects_MULTI(self, modpath, major, recursive, methods=None): + """ + All args are as for the `get_existing_objects()` method, except for `methods`, + where you can pass a list of integers, being the numbers of the methods you + want to employ. If more than one, we will compare their results. + """ + if methods is None: + methods = [0, 1] + comparing = len(methods) > 1 + + N, R = [], [] + for m in methods: + get_existing_j_objects = getattr(self, f'get_existing_j_objects_{m}') + + existing_j_nodes, existing_j_relns = get_existing_j_objects( + modpath, major, recursive, print_times=comparing) + + # Convert from j-nodes to k-nodes. + existing_k_nodes = {} + for j in existing_j_nodes: + k = make_kNode_from_jNode(j) + existing_k_nodes[k.uid] = k + + # Convert from j-relns to k-relns. + existing_k_relns = {} + for j in existing_j_relns: + k = make_kReln_from_jReln(j) + # Reject inferred relations. + if k.reln_type in IndexType.INFERRED_RELNS: + continue + existing_k_relns[k.uid] = k + + N.append(existing_k_nodes) + R.append(existing_k_relns) + + N0, R0 = N[0], R[0] + + if comparing: + # Did we get the same results? + for m in methods[1:]: + print(f'\nComparing method {m}') + N, R = N[m], R[m] + k1 = set(N.keys()) + k0 = set(N0.keys()) + if k1 != k0: + print(' Got different set of node keys.') + else: + print(' Got same set of node keys.') + if set(R.keys()) != set(R0.keys()): + print(' Got different set of reln keys.') + else: + print(' Got same set of reln keys.') + + return N0, R0 + + def get_existing_j_objects_0(self, modpath, major, recursive, print_times=False): + """ + Use `starting_with()` to locate all objects having modpath with the right prefix. + """ + + t0 = time.time() + existing_j_nodes = covers( major, self.g.V().has('modpath', modpath) ).element_map().to_list() @@ -128,12 +236,9 @@ def get_existing_objects(self, modpath, major, recursive): self.g.V().has('modpath', TextP.starting_with(basepath)) ).element_map().to_list() existing_j_nodes += existing_j_nodes_under_module - # Convert from j-nodes to k-nodes. - existing_k_nodes = {} - for j in existing_j_nodes: - k = make_kNode_from_jNode(j) - existing_k_nodes[k.uid] = k - # Relations + + t1 = time.time() + existing_j_relns = edge_info(covers( major, self.g.E().has('modpath', modpath) )).to_list() @@ -144,15 +249,56 @@ def get_existing_objects(self, modpath, major, recursive): self.g.E().has('modpath', TextP.starting_with(basepath)) )).to_list() existing_j_relns += existing_j_relns_under_module - # Convert from j-relns to k-relns. - existing_k_relns = {} - for j in existing_j_relns: - k = make_kReln_from_jReln(j) - # Reject inferred relations. - if k.reln_type in IndexType.INFERRED_RELNS: - continue - existing_k_relns[k.uid] = k - return existing_k_nodes, existing_k_relns + + t2 = time.time() + + if print_times: + print('get_existing_j_objects_0 internal times:') + print(f' {t1 - t0}') + print(f' {t2 - t1}') + + return existing_j_nodes, existing_j_relns + + def get_existing_j_objects_1(self, modpath, major, recursive, print_times=False): + """ + First do a BFS with `repeat()` to determine all the modpaths that begin with the + right prefix, and then return all objects having any of these modpaths. + """ + t0 = time.time() + + if not recursive: + modpaths = [modpath] + else: + modpaths = covers( + major, + covers( + major, self.g.V().has('libpath', modpath) + ).emit().repeat(__.in_(IndexType.UNDER).has_label(IndexType.MODULE)) + ).values('modpath').to_list() + + t1 = time.time() + + existing_j_nodes = covers( + major, self.g.V().has('modpath', P.within(modpaths)) + ).element_map().to_list() + + t2 = time.time() + + existing_j_relns = edge_info(covers( + major, self.g.E().has('modpath', P.within(modpaths)) + )).to_list() + + t3 = time.time() + + if print_times: + print('get_existing_j_objects_1 internal times:') + print(f' {t1 - t0} ({len(modpaths)} modpaths)') + print(f' {t2 - t1} ({len(existing_j_nodes)} nodes)') + print(f' {t3 - t2} ({len(existing_j_relns)} relns)') + + return existing_j_nodes, existing_j_relns + + ################################################################################ def _get_origins_internal(self, label, libpaths, major0): M = lps_covers(libpaths, major0, self.g.V().has_label(label)). \ @@ -193,6 +339,9 @@ def _get_deductive_nbrs_internal(self, libpaths, major0): ).values('libpath').to_set() def _get_deduction_closure_internal(self, libpaths, major0): + # Note: As noted earlier in this module, a `repeat()` that only follows + # outgoing UNDER edges has no need of a `limit(1)`, since there can be + # no branching on such paths. res = among_lps(libpaths, self.g.V()).as_('u') \ .repeat( covers(major0, __.out_e(IndexType.UNDER)).in_v() @@ -236,6 +385,9 @@ def get_results_relying_upon(self, deducpath, major, realm=None): if realm is None: realm = '.'.join(deducpath.split('.')[:3]) realmbase = realm + '.' + # Note: As noted earlier in this module, a `repeat()` that only follows + # outgoing UNDER edges has no need of a `limit(1)`, since there can be + # no branching on such paths. return covers( major0, lp_covers(deducpath, major0, self.g.V()).in_e(IndexType.GHOSTOF) @@ -418,14 +570,18 @@ def _object_is_built(self, libpath, version): def check_approvals_under_anno(self, annopath, version): major0 = self.adaptall(version) - basepath = annopath + '.' + # Note: Here our goal is to find *all* widgets under a given annotation, + # so we do *not* want a `limit(1)` on this `repeat()`. elt_maps = covers( - major0, self.g.V().has_label(IndexType.WIDGET) - ).has('libpath', TextP.starting_with(basepath)). \ - element_map('libpath', IndexType.P_APPROVALS).to_list() + major0, + lp_covers( + annopath, major0, self.g.V() + ).repeat(__.in_(IndexType.UNDER)) + .until(__.has_label(IndexType.WIDGET)) + ).element_map('libpath', IndexType.P_APPROVALS).to_list() approved = [] for elt_map in elt_maps: - j = elt_map.get('approvals') + j = elt_map.get(IndexType.P_APPROVALS) if j is not None: approvals = json.loads(j) if approvals.get(version, False): diff --git a/server/tests/resources/repo/moo/bar/v0.3.4/__.pfsc b/server/tests/resources/repo/moo/bar/v0.3.4/__.pfsc new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/test_cf.py b/server/tests/test_cf.py index b83c7fb..adffcc5 100644 --- a/server/tests/test_cf.py +++ b/server/tests/test_cf.py @@ -14,8 +14,11 @@ # limitations under the License. # # --------------------------------------------------------------------------- # +import os import json +import pytest + from tests import handleAsJson from pfsc.constants import ISE_PREFIX, IndexType @@ -57,6 +60,8 @@ def test_cf_out(client, repos_ready): ] +# See `tests.util.gather_repo_info()` function, on the reason for the conditional skip here. +@pytest.mark.skipif(bool(int(os.getenv("DUPLICATE_TEST_HIST_LIT", 0))), reason="fails when test.hist.lit has a v1") def test_cf_in(client, repos_ready): libpath = 'test.hist.lit.K.ummer.Cr040_08.Pf' vers = 'v0.0.0' diff --git a/server/tests/util/__init__.py b/server/tests/util/__init__.py index ee91b37..936d517 100644 --- a/server/tests/util/__init__.py +++ b/server/tests/util/__init__.py @@ -80,6 +80,21 @@ def gather_repo_info(): vers_dirs.sort(key=lambda d: VersionTag(d)) # For tag names, pad pre-multi-versioning names with zeros. tag_names = [f'{vers}.0.0' if vers.find('.') < 0 else vers for vers in vers_dirs] + + # Set env var DUPLICATE_TEST_HIST_LIT=1 to activate this. + if bool(int(os.getenv("DUPLICATE_TEST_HIST_LIT", 0))): + # This is a hack, to provide us with a second "version" of the `test.hist.lit` + # repo, under `v1`. The content is actually the same. The purpose of doing this + # is to be able to test methods like `pfsc.gdb.reader.GraphReader.get_existing_objects()`, + # which only really have work to do when indexing a *second* version, on a fairly + # large repo. We do it as a hack instead of simply adding a second version + # of that repo in the source code because, apart from efficiency experiments like + # this one, I don't think we want to slow down our regular unit tests with such + # a thing. + if user == 'hist' and proj == 'lit': + vers_dirs.append('v0') + tag_names.append('v1.0.0') + REPOS.append(Repo(root_dir, user, proj, vers_dirs, tag_names)) return REPOS From 773c986b30ffd49bb0ac0d5291c6fef40511270d Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 04:12:06 -0500 Subject: [PATCH 11/35] Use GremLite instead of RedisGraph in the OCA --- manage/conf/base_conf.py | 4 ++-- manage/pise_python_vers.env | 2 +- manage/tools/build.py | 6 +++--- manage/tools/util.py | 2 +- manage/topics/pfsc/__init__.py | 4 ++-- manage/topics/pfsc/templates/Dockerfile.oca | 20 +++++-------------- .../pfsc/templates/Dockerfile.oca_final_setup | 10 ++-------- manage/topics/redis/__init__.py | 7 +++++++ manage/topics/redis/templates/redis.ini | 9 +++++++++ server/config.py | 2 +- 10 files changed, 33 insertions(+), 33 deletions(-) create mode 100644 manage/topics/redis/templates/redis.ini diff --git a/manage/conf/base_conf.py b/manage/conf/base_conf.py index bc3da37..71bf21a 100644 --- a/manage/conf/base_conf.py +++ b/manage/conf/base_conf.py @@ -176,8 +176,8 @@ # # These are the default docker image tags that will be used, for various # services, or as starting points for building our own docker images. -PYTHON_IMAGE_TAG = f'{PYTHON_VERSION}-slim-bullseye' -REDIS_IMAGE_TAG = '6.2.1' +PYTHON_IMAGE_TAG = f'{PYTHON_VERSION}-slim-bookworm' +REDIS_IMAGE_TAG = '8.4.0-bookworm' REDISGRAPH_IMAGE_TAG = '6.2.6-v6' NEO4J_IMAGE_TAG = '4.4.46' GREMLIN_SERVER_IMAGE_TAG = '3.7.5' diff --git a/manage/pise_python_vers.env b/manage/pise_python_vers.env index 5a3e1ff..4a735a4 100644 --- a/manage/pise_python_vers.env +++ b/manage/pise_python_vers.env @@ -1,2 +1,2 @@ # The Python version currently used in PISE is: -PISE_PYTHON_VERS=3.11.13 +PISE_PYTHON_VERS=3.11.14 diff --git a/manage/tools/build.py b/manage/tools/build.py index 1e34473..771947d 100644 --- a/manage/tools/build.py +++ b/manage/tools/build.py @@ -307,7 +307,7 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str): from topics.pfsc import write_oca_eula_file from topics.pfsc import write_worker_and_web_supervisor_ini from topics.pfsc import write_proofscape_oca_dockerfile - from topics.redis import write_redisgraph_ini + from topics.redis import write_redis_ini with tempfile.TemporaryDirectory(dir=SRC_TMP_ROOT) as tmp_dir_name: with open(os.path.join(tmp_dir_name, 'license_info.json'), 'w') as f: f.write(json.dumps(license_info, indent=4)) @@ -322,8 +322,8 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str): ini = write_worker_and_web_supervisor_ini( worker=False, web=True, use_venv=False, oca=True) f.write(ini) - with open(os.path.join(tmp_dir_name, 'redisgraph.ini'), 'w') as f: - ini = write_redisgraph_ini(use_conf_file=True) + with open(os.path.join(tmp_dir_name, 'redis.ini'), 'w') as f: + ini = write_redis_ini(use_conf_file=True) f.write(ini) with open(os.path.join(tmp_dir_name, 'oca_version.txt'), 'w') as f: f.write(tag) diff --git a/manage/tools/util.py b/manage/tools/util.py index 0b5dc91..82769b1 100644 --- a/manage/tools/util.py +++ b/manage/tools/util.py @@ -180,7 +180,7 @@ def get_redis_server_version_for_oca(): """ Get the version number of redis-server that is installed in the OCA image. """ - cmd = f'docker run --rm --entrypoint=bash redis/redis-stack-server:{pfsc_conf.REDISGRAPH_IMAGE_TAG} -c "redis-server --version"' + cmd = f'docker run --rm --entrypoint=bash redis:{pfsc_conf.REDIS_IMAGE_TAG} -c "redis-server --version"' out = subprocess.check_output(cmd, shell=True) text = out.decode() diff --git a/manage/topics/pfsc/__init__.py b/manage/topics/pfsc/__init__.py index bc17c7d..12a6221 100644 --- a/manage/topics/pfsc/__init__.py +++ b/manage/topics/pfsc/__init__.py @@ -200,7 +200,7 @@ def write_proofscape_oca_dockerfile(tmp_dir_name, demos=False): ) startup_system = write_startup_system( '/home/pfsc', numbered_inis={ - 100: 'redisgraph', + 100: 'redis', 200: 'pfsc', }, tmp_dir_name=tmp_dir_name ) @@ -213,7 +213,7 @@ def write_proofscape_oca_dockerfile(tmp_dir_name, demos=False): template = jinja_env.get_template('Dockerfile.oca') df = template.render( python_image_tag=conf.PYTHON_IMAGE_TAG, - redisgraph_image_tag=conf.REDISGRAPH_IMAGE_TAG, + redis_image_tag=conf.REDIS_IMAGE_TAG, platform=conf.DOCKER_PLATFORM, pfsc_install=pfsc_install, startup_system=startup_system, diff --git a/manage/topics/pfsc/templates/Dockerfile.oca b/manage/topics/pfsc/templates/Dockerfile.oca index 4e76694..08c4fce 100644 --- a/manage/topics/pfsc/templates/Dockerfile.oca +++ b/manage/topics/pfsc/templates/Dockerfile.oca @@ -14,27 +14,17 @@ # limitations under the License. # # --------------------------------------------------------------------------- # -FROM --platform={{platform}} redis/redis-stack-server:{{redisgraph_image_tag}} AS rg +FROM --platform={{platform}} redis:{{redis_image_tag}} AS redis_image FROM --platform={{platform}} python:{{python_image_tag}} AS basis ARG DEBIAN_FRONTEND=noninteractive -# `libgomp1` is needed by redisgraph. # `sudo` and `less` are installed for dev purposes, and user `pfsc` (though it # isn't added until later in the Dockerfile) is given passwordless sudo. RUN apt-get update \ - && apt-get install -y --no-install-recommends libgomp1 sudo less \ + && apt-get install -y --no-install-recommends sudo less \ && rm -rf /var/lib/apt/lists/* \ - && echo "pfsc ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers \ - && mkdir -p /usr/lib/redis/modules -COPY --from=rg /opt/redis-stack/lib/redisgraph.so /usr/lib/redis/modules -# We don't need redisearch.so for our purposes; we load it only so that we -# can successfully read RDB files that were dumped by a Redis that does have -# redisearch.so. See -# The latter occurs e.g. when the dump file was created by pise/server unit -# tests, running against a redis/redis-stack-server image. -COPY --from=rg /opt/redis-stack/lib/redisearch.so /usr/lib/redis/modules -COPY --from=rg /usr/bin/redis-server /usr/local/bin -# Note: Could save 6MB by ignoring redis-cli, which is just a debugging tool. -COPY --from=rg /usr/bin/redis-cli /usr/local/bin + && echo "pfsc ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers +COPY --from=redis_image /usr/local/bin/redis-server /usr/local/bin + # For the sake of casual users, I think it's important that the Proofscape ISE # app port 7372 get top billing in Docker Dashboard, in particular in the # "Run" dialog. Therefore, we don't expose 6379. Casual users shouldn't have diff --git a/manage/topics/pfsc/templates/Dockerfile.oca_final_setup b/manage/topics/pfsc/templates/Dockerfile.oca_final_setup index 8726c51..3718a7c 100644 --- a/manage/topics/pfsc/templates/Dockerfile.oca_final_setup +++ b/manage/topics/pfsc/templates/Dockerfile.oca_final_setup @@ -21,14 +21,8 @@ USER pfsc WORKDIR {{final_workdir}} # redis.conf -# In particular, `save 1 1` means that, when there are any changes in our GDB -# (RedisGraph), it will be at most 1 second before Redis initiates a BGSAVE to -# commit them to disk. Combined with use of `pfsc.gdb.cypher.rg.redis_bg_save()` -# in pfsc-server, this should achieve pretty robust persistence. -RUN echo "loadmodule /usr/lib/redis/modules/redisgraph.so" >> redis.conf \ - && echo "loadmodule /usr/lib/redis/modules/redisearch.so" >> redis.conf \ - && echo "dir /proofscape/graphdb/re" >> redis.conf \ - && echo "save 1 1" >> redis.conf +RUN echo "# For now, we need no special config." >> redis.conf +RUN echo "# If and when we do, use this file." >> redis.conf # Use the One-Container App config: ENV FLASK_CONFIG OCA diff --git a/manage/topics/redis/__init__.py b/manage/topics/redis/__init__.py index 6e4748b..134e079 100644 --- a/manage/topics/redis/__init__.py +++ b/manage/topics/redis/__init__.py @@ -31,6 +31,13 @@ def write_redisgraph_ini(use_conf_file=True): ) +def write_redis_ini(use_conf_file=True): + template = jinja_env.get_template('redis.ini') + return template.render( + use_conf_file=use_conf_file, + ) + + REDIS_DOCKERFILE_TPLT = jinja2.Template("""\ FROM redis:{{redis_image_tag}} COPY {{tmp_dir_name}}/redis.conf /usr/local/etc/redis/redis.conf diff --git a/manage/topics/redis/templates/redis.ini b/manage/topics/redis/templates/redis.ini new file mode 100644 index 0000000..c0097a3 --- /dev/null +++ b/manage/topics/redis/templates/redis.ini @@ -0,0 +1,9 @@ +[program:redis] +{% if use_conf_file %} +command=redis-server /home/pfsc/redis.conf +{% else %} +command=redis-server +{% endif %} +priority=100 +user=pfsc +autostart=true diff --git a/server/config.py b/server/config.py index 1f163d7..00aa6a3 100644 --- a/server/config.py +++ b/server/config.py @@ -661,7 +661,7 @@ class OcaConfig(DockerDevConfig): RECORD_PER_USER_TRUST_SETTINGS = True REDIS_URI = "redis://localhost:6379" SOCKETIO_MESSAGE_QUEUE = "redis://localhost:6379" - GRAPHDB_URI = "redis://localhost:6379" + GRAPHDB_URI = "file:///home/pfsc/proofscape/graphdb/gl/gremlite.db" # The app URL prefix is important, so that the URL at which the ISE loads, # `localhost:7372/ProofscapeISE` is recognizable by the PBE (Proofscape From 04033ef2219cd464cfdd7510e45dc2c2331c6c7f Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 04:54:24 -0500 Subject: [PATCH 12/35] Form gremlite path dirs automatically --- server/pfsc/gdb/__init__.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/server/pfsc/gdb/__init__.py b/server/pfsc/gdb/__init__.py index dc31844..1ae9492 100644 --- a/server/pfsc/gdb/__init__.py +++ b/server/pfsc/gdb/__init__.py @@ -14,6 +14,8 @@ # limitations under the License. # # --------------------------------------------------------------------------- # +import pathlib + from flask import current_app from flask import g as flask_g from gremlin_python.process.anonymous_traversal import traversal @@ -50,9 +52,15 @@ def get_gdb(): uri = current_app.config["GRAPHDB_URI"] # Decide by the form of the URI which graph database system we are using. protocol = uri.split(":")[0] - if uri.endswith('/gremlin') or protocol == 'file': + if protocol == 'file' or (protocol in ['ws', 'wss'] and uri.endswith('/gremlin')): + # It looks like you are using Gremlin. if protocol == 'file': - path = uri[7:] + # We assume you want to use GremLite. + path = pathlib.Path(uri[7:]) + # We ensure the named directory and any missing parent dirs exist. + path_dir = path.parent + if not path_dir.exists(): + path_dir.mkdir(parents=True) # Set these True to log low level SQLite usage: log_plans = False check_qqc_patterns = False @@ -62,6 +70,8 @@ def get_gdb(): remote = SQLiteConnection(path, autocommit=False, log_plans=log_plans, check_qqc_patterns=check_qqc_patterns) else: + # We assume you are connecting to a Gremlin server. + # # Starting with gremlinpython==3.6.1, we have to explicitly request the # `GraphSONSerializersV3d0` message serializer. This was the default in 3.6.0, # but in 3.6.1 they changed the default to `GraphBinarySerializersV1`. From 58abc07038242825ba8f5789cf082193d502f974 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 04:55:11 -0500 Subject: [PATCH 13/35] Improve documentation of `GRAPHDB_URI` config var --- server/config.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/server/config.py b/server/config.py index 00aa6a3..0a0478f 100644 --- a/server/config.py +++ b/server/config.py @@ -303,6 +303,31 @@ class Config: REDIS_URI = os.getenv("REDIS_URI") + # pise-server records data in a graph database (GDB). It is up to you to choose the + # database system, and ensure that it is available. Use the `GRAPHDB_URI` config var + # to tell pise-server which system to use and how to connect to it. Your options are + # as follows: + # + # * Any URI with scheme `file`, such as `file:///filesystem/path/to/my_sqlite_database_file.db` + # In this case, pise-server will assume that you want to use GremLite as your + # GDB. Support is built-in, and you do not need to provide anything else. + # The file you name either should not yet exist, or may already exist iff it is + # from previous use by pise-server. If the directory you name does not yet exist, then + # it and any missing parent directories will be automatically formed by pise-server. + # + # * Any URI ending with `/gremlin` and using scheme `ws` or `wss` + # pise-server will assume you want a websocket connection to a Gremlin Server, + # and it will use the `gremlinpython` package to connect to that. + # + # * Any URI with scheme `redis` or `rediss` + # pise-server will assume the URI points to an instance of Redis server that has + # loaded the RedisGraph module, and it will use RedisGraph. + # + # * Any URI with scheme `bolt` or `neo4j` + # pise-server will assume the URI points to an instance of Neo4j server, and it + # will connect using the `neo4j` Python package. + # + # Any other URI format will raise an exception. GRAPHDB_URI = os.getenv("GRAPHDB_URI") # Username and password for the GDB are optional. GDB_USERNAME = os.getenv("GDB_USERNAME") or '' @@ -316,14 +341,14 @@ class Config: # atomicity, and in some cases may make operations significantly faster (for example, # GremLite is much, much faster when we group our changes using transactions). # - # The current design in pise/server is influenced by the set of GDB systems against + # The current design in pise-server is influenced by the set of GDB systems against # which it has ever been tested. Among Cypher systems, there are only two: Neo4j, # and RedisGraph. The former supports transactions, the latter does not. As a consequence, - # pise/server will always use transactions when using Cypher, unless connecting to RedisGraph. + # pise-server will always use transactions when using Cypher, unless connecting to RedisGraph. # Thus, the `USE_TRANSACTIONS` config var currently has no effect when connecting to any # Cypher GDB. # - # If you are trying to use pise/server with a Cypher GDB other than RedisGraph that does + # If you are trying to use pise-server with a Cypher GDB other than RedisGraph that does # not support transactions, please open an issue at the PISE GitHub page. # # On the Gremlin side, we have tested against GremLite, where transactions are supported, From 5338c0fcbfe875abe0665c86784ddd66387ea90a Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 04:56:18 -0500 Subject: [PATCH 14/35] Add gremlite as deployment option --- manage/tools/deploy/__init__.py | 6 +++--- manage/tools/deploy/services.py | 26 +++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/manage/tools/deploy/__init__.py b/manage/tools/deploy/__init__.py index 79e1663..7bf5f8c 100644 --- a/manage/tools/deploy/__init__.py +++ b/manage/tools/deploy/__init__.py @@ -98,8 +98,8 @@ def production(gdb, workers, demos, dump_dc, dirname, official, pfsc_tag): @deploy.command() @click.option('--gdb', - default=GdbCode.RE, prompt='Graph database (re, nj, tk, ja, np)', - help='List one or more graph DBs. re=RedisGraph, nj=Neo4j, tk=TinkerGraph, ja=JanusGraph, np=Neptune') + default=GdbCode.GL, prompt='Graph database (gl, re, nj, tk, ja, np)', + help='List one or more graph DBs. gl=GremLite, re=RedisGraph, nj=Neo4j, tk=TinkerGraph, ja=JanusGraph, np=Neptune') @click.option('--pfsc-tag', default='testing', prompt='pise-server image tag', help='Use `pise-server:TEXT` docker image.') @click.option('--frontend-tag', @@ -597,7 +597,7 @@ def write_gdb_dot_env(d, gdb, uri_lookup_method, comment=False): 'out': i > 0 or GdbCode.requires_manual_URI(code), } if comment: - v['comment'] = GdbCode.service_name(code) + ":" + v['comment'] = GdbCode.comment_name(code) + ":" d[f"GRAPHDB_URI_{i}"] = v d['post-gdb-block'] = {'comment': ''} diff --git a/manage/tools/deploy/services.py b/manage/tools/deploy/services.py index 4b15fe4..8cac03f 100644 --- a/manage/tools/deploy/services.py +++ b/manage/tools/deploy/services.py @@ -22,6 +22,8 @@ class GdbCode: + # GremLite + GL = 'gl' # RedisGraph RE = 're' # Neo4j @@ -33,7 +35,7 @@ class GdbCode: # Neptune NP = 'np' - all = [RE, NJ, TK, JA, NP] + all = [GL, RE, NJ, TK, JA, NP] # Those that are deployed via container: via_container = [RE, NJ, TK, JA] @@ -67,6 +69,12 @@ def standard_port(cls, code): cls.NP: 8182, }[code] + @classmethod + def comment_name(cls, code): + if code == cls.GL: + return 'GremLite' + return cls.service_name(code) + @classmethod def service_name(cls, code): return { @@ -89,13 +97,17 @@ def uri_path(cls, code): @classmethod def localhost_URI(cls, code): - if code == cls.NP: + if code == cls.GL: + return cls.GremLite_URI(PFSC_ROOT) + elif code == cls.NP: return cls.Neptune_URI() return f'{cls.protocol(code)}://localhost:{cls.host_port(code)}{cls.uri_path(code)}' @classmethod def docker_URI(cls, code): - if code == cls.NP: + if code == cls.GL: + return cls.GremLite_URI('/proofscape') + elif code == cls.NP: return cls.Neptune_URI() return f'{cls.protocol(code)}://{cls.service_name(code)}:{cls.standard_port(code)}{cls.uri_path(code)}' @@ -108,6 +120,11 @@ def service_defn_writer(cls, code): cls.JA: janusgraph, }[code] + @classmethod + def GremLite_URI(cls, pfsc_root_path): + db_file_path = pathlib.Path(pfsc_root_path) / 'graphdb/gl/gremlite.db' + return f'file://{db_file_path}' + @classmethod def Neptune_URI(cls): prefix = conf.AWS_NEPTUNE_URI_PREFIX or 'XXXXXXXXXXXXXXXXXXXXXXXXXXX' @@ -288,6 +305,9 @@ def pise_server(deploy_dir_path, mode, flask_config, tag='latest', if mode == 'websrv': d['depends_on'].extend(GdbCode.service_name(code) for code in gdb if code in GdbCode.via_container) d['depends_on'].extend([f'pfscwork{n}' for n in range(workers)]) + if GdbCode.GL in gdb: + direc = 'graphdb/gl' + d['volumes'].append(f'{get_proofscape_subdir_abs_fs_path_on_host(direc, altdir=altdir)}:/proofscape/{direc}') if conf.EMAIL_TEMPLATE_DIR: d['volumes'].append(f'{resolve_fs_path("EMAIL_TEMPLATE_DIR")}:/home/pfsc/proofscape/src/_email_templates:ro') if mount_code: From 48dbc9ed48b6eeb631e9d996618bd80e408742b4 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 05:12:08 -0500 Subject: [PATCH 15/35] Bugfix in `pfsc.build.OriginInjectionVisitor` This was a funny issue that could have been resolved in several different ways. See notes from 260118 on ghost node issue. Was exposed by use of gremlite, which does not support the `has_label(None)` case that *is* supported in TinkerGraph. --- server/pfsc/build/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/pfsc/build/__init__.py b/server/pfsc/build/__init__.py index dd83373..3e8e744 100644 --- a/server/pfsc/build/__init__.py +++ b/server/pfsc/build/__init__.py @@ -340,7 +340,7 @@ def __call__(self, item): if isinstance(item, GhostNode): real_obj = item.realObj() - if not real_obj.getOrigin(): + if self.takes_origin(real_obj) and not real_obj.getOrigin(): realpath = real_obj.getLibpath() if realpath in self.lp2origin: real_obj.setOrigin(self.lp2origin[realpath]) From 03c3f1732a5b48d68c6fee5bb065a5a9ba243bbf Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 08:04:03 -0500 Subject: [PATCH 16/35] Remove unnecessary `mkdir graphdb/re` --- manage/topics/pfsc/templates/Dockerfile.pfsc | 2 +- manage/topics/pfsc/templates/Dockerfile.startup_system | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manage/topics/pfsc/templates/Dockerfile.pfsc b/manage/topics/pfsc/templates/Dockerfile.pfsc index 4baa786..ea85c30 100644 --- a/manage/topics/pfsc/templates/Dockerfile.pfsc +++ b/manage/topics/pfsc/templates/Dockerfile.pfsc @@ -20,7 +20,7 @@ RUN adduser --disabled-password --gecos "" pfsc \ && sed -i -e "s/#alias ll=/alias ll=/" /home/pfsc/.bashrc \ && sed -i -e "s%# \(export\|eval\|alias l\)%\1%" /root/.bashrc \ - && bash -c "mkdir -p /proofscape/{lib,build/_sphinx,graphdb/re,deploy,PDFLibrary}" + && bash -c "mkdir -p /proofscape/{lib,build/_sphinx,graphdb,deploy,PDFLibrary}" {% else %} RUN adduser -D pfsc \ && mkdir /proofscape diff --git a/manage/topics/pfsc/templates/Dockerfile.startup_system b/manage/topics/pfsc/templates/Dockerfile.startup_system index d111d1f..8ced866 100644 --- a/manage/topics/pfsc/templates/Dockerfile.startup_system +++ b/manage/topics/pfsc/templates/Dockerfile.startup_system @@ -39,7 +39,7 @@ RUN chown -R pfsc:pfsc super ARG STARTUP=/usr/local/bin/startup RUN echo "#!/bin/bash" > $STARTUP \ {% if ensure_dirs %} \ - && echo "mkdir -p /proofscape/{lib,build,graphdb/re,deploy,PDFLibrary}" >> $STARTUP \ + && echo "mkdir -p /proofscape/{lib,build,graphdb,deploy,PDFLibrary}" >> $STARTUP \ {% endif %} \ && echo "exec supervisord -n -c {{dir_where_startup_system_lives}}/super/supervisord.conf" >> $STARTUP \ && chmod +x $STARTUP From 1269b78e2aea905515bc2943065b57bfcfd999c4 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 10:52:11 -0500 Subject: [PATCH 17/35] Include GremLite in build-and-test workflow --- .github/workflows/pise-build-and-test.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index d2f1a77..9f54a48 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -43,6 +43,10 @@ jobs: server-unit-tests: runs-on: ubuntu-22.04 services: + redis: + image: "redis:8.4.0-bookworm" + ports: + - "6379:6379" redisgraph: image: "redis/redis-stack-server:6.2.6-v6" ports: @@ -53,13 +57,22 @@ jobs: - "8182:8182" strategy: matrix: - graphdb: ['Cypher: RedisGraph', 'Gremlin: TinkerGraph'] + include: + - name: GremLite + graphdb_uri: ${{ format('file://{0}/graphdb/gl/gremlite.db', github.workspace) }} + redis_uri: redis://localhost:6379 + - name: RedisGraph + graphdb_uri: redis://localhost:6381 + redis_uri: redis://localhost:6381 + - name: TinkerGraph + graphdb_uri: ws://localhost:8182/gremlin + redis_uri: redis://localhost:6379 defaults: run: shell: bash env: - REDIS_URI: redis://localhost:6381 - GRAPHDB_URI: ${{ startsWith(matrix.graphdb, 'C') && 'redis://localhost:6381' || 'ws://localhost:8182/gremlin' }} + REDIS_URI: ${{ matrix.redis_uri }} + GRAPHDB_URI: ${{ matrix.graphdb_uri }} PFSC_LIB_ROOT: ${{ format('{0}/lib', github.workspace) }} PFSC_BUILD_ROOT: ${{ format('{0}/build', github.workspace) }} SECRET_KEY: fixed_value_for_testing From 9892403985a6cbfd341c6b0e20d96155b52cb9f1 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 10:53:18 -0500 Subject: [PATCH 18/35] Add news fragments regarding gremlite --- changelog.d/99.changed.txt | 4 ++++ changelog.d/99.improved.txt | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/99.changed.txt create mode 100644 changelog.d/99.improved.txt diff --git a/changelog.d/99.changed.txt b/changelog.d/99.changed.txt new file mode 100644 index 0000000..4bcc36e --- /dev/null +++ b/changelog.d/99.changed.txt @@ -0,0 +1,4 @@ +The [PISE one-container app Docker image](https://hub.docker.com/r/proofscape/pise) now +uses `GremLite` instead of `RedisGraph` as its built-in graph database system. +Users of previous versions of the app will have to rebuild all Proofscape repos in their +library, as the new app will not be able to read the old graph database files. diff --git a/changelog.d/99.improved.txt b/changelog.d/99.improved.txt new file mode 100644 index 0000000..522dab8 --- /dev/null +++ b/changelog.d/99.improved.txt @@ -0,0 +1,2 @@ +`pise/server` now supports the [GremLite](https://pypi.org/project/gremlite/) +graph database system. From bddc878e20383e3322ddfc75ba8d5bc640a48803 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 11:41:42 -0500 Subject: [PATCH 19/35] Remove default to redisgraph in `pise_server()` service definition I couldn't find any good reason to have the `gdb` arg default to `[GdbCode.RE]`, so I made `gdb` a required arg instead of keyword. --- manage/tools/deploy/__init__.py | 12 ++++++------ manage/tools/deploy/services.py | 9 +++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manage/tools/deploy/__init__.py b/manage/tools/deploy/__init__.py index 7bf5f8c..de2ffac 100644 --- a/manage/tools/deploy/__init__.py +++ b/manage/tools/deploy/__init__.py @@ -170,7 +170,7 @@ def generate(gdb, pfsc_tag, frontend_tag, oca_tag, official, workers, demos, mou raise click.UsageError(f'Legal GDB codes are: {", ".join(GdbCode.all)}') if len(gdb) > len(s): raise click.UsageError('Cannot repeat graph database selections.') - if no_redis and s != {'re'}: + if no_redis and s != {GdbCode.RE}: raise click.UsageError('RedisGraph must be sole GDB selection, when using --no-redis.') dirname_prefix = 'production_' if production_mode else None @@ -180,7 +180,7 @@ def generate(gdb, pfsc_tag, frontend_tag, oca_tag, official, workers, demos, mou # admin shell script admin_sh_script = write_admin_sh_script( - new_dir_name, new_dir_path, pfsc_tag, flask_config, + new_dir_name, new_dir_path, pfsc_tag, flask_config, gdb, demos=demos, mount_code=mount_code, mount_pkg=mount_pkg, official=official, no_redis=no_redis ) @@ -673,14 +673,14 @@ def write_dc_script(deploy_dir_name): def write_admin_sh_script( - deploy_dir_name, deploy_dir_path, pfsc_tag, flask_config, + deploy_dir_name, deploy_dir_path, pfsc_tag, flask_config, gdb, demos=False, mount_code=False, mount_pkg=None, official=False, no_redis=False ): # Want all the same bind mounts that are used in a pfsc worker container, # so that admin can do anything a worker can do. d = services.pise_server( - deploy_dir_path, 'worker', flask_config, tag=pfsc_tag, + deploy_dir_path, 'worker', flask_config, gdb, tag=pfsc_tag, demos=demos, mount_code=mount_code, mount_pkg=mount_pkg, official=official, no_redis=no_redis ) @@ -765,8 +765,8 @@ def write_docker_compose_yaml(deploy_dir_name, deploy_dir_path, gdb, pfsc_tag, f s_app = {} def write_pfsc_service(cmd): - return services.pise_server(deploy_dir_path, cmd, flask_config, - tag=pfsc_tag, gdb=gdb, workers=workers, demos=demos, + return services.pise_server(deploy_dir_path, cmd, flask_config, gdb, + tag=pfsc_tag, workers=workers, demos=demos, mount_code=mount_code, mount_pkg=mount_pkg, official=official, altdir=altdir, lib_vol=lib_vol, build_vol=build_vol, no_redis=no_redis) diff --git a/manage/tools/deploy/services.py b/manage/tools/deploy/services.py index 8cac03f..b0f3196 100644 --- a/manage/tools/deploy/services.py +++ b/manage/tools/deploy/services.py @@ -257,8 +257,8 @@ def get_proofscape_subdir_abs_fs_path_on_host(subdir_name, altdir=None): return resolve_pfsc_root_subdir(subdir_name) -def pise_server(deploy_dir_path, mode, flask_config, tag='latest', - gdb=None, workers=1, demos=False, +def pise_server(deploy_dir_path, mode, flask_config, gdb, tag='latest', + workers=1, demos=False, mount_code=False, mount_pkg=None, official=False, altdir=None, lib_vol=None, build_vol=None, @@ -277,8 +277,10 @@ def pise_server(deploy_dir_path, mode, flask_config, tag='latest', } if no_redis: + # When this is set, gdb is required to be equal to [GdbCode.RE]. + assert gdb == [GdbCode.RE] if mode == 'websrv': - # redisgraph will be added below, as GDB dependency + # redisgraph will be added below, as GDB dependency, so we don't add it here. pass else: d['depends_on'].append('redisgraph') @@ -301,7 +303,6 @@ def pise_server(deploy_dir_path, mode, flask_config, tag='latest', }[mode] d['environment'][mode_env_var] = 1 - gdb = gdb or [GdbCode.RE] if mode == 'websrv': d['depends_on'].extend(GdbCode.service_name(code) for code in gdb if code in GdbCode.via_container) d['depends_on'].extend([f'pfscwork{n}' for n in range(workers)]) From f4d4ada8a2c305de9e4a78779d1bfcc2d3ddaa6c Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 16:06:11 -0500 Subject: [PATCH 20/35] Extend support for `--gdb-vol` switch to GremLite We also improve the behavior for unsupported cases, now printing a helpful error message. --- manage/tools/deploy/__init__.py | 13 ++++++++----- manage/tools/deploy/services.py | 28 +++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/manage/tools/deploy/__init__.py b/manage/tools/deploy/__init__.py index de2ffac..f9b11cd 100644 --- a/manage/tools/deploy/__init__.py +++ b/manage/tools/deploy/__init__.py @@ -130,7 +130,7 @@ def production(gdb, workers, demos, dump_dc, dirname, official, pfsc_tag): @click.option('--dummy', is_flag=True, help='Write a docker compose yml for a dummy deployment (Hello World web app).') @click.option('--lib-vol', help="A pre-existing docker volume to be mounted to /proofscape/lib") @click.option('--build-vol', help="A pre-existing docker volume to be mounted to /proofscape/build") -@click.option('--gdb-vol', help="A pre-existing docker volume for the graph db. Only RedisGraph currently supported.") +@click.option('--gdb-vol', help="A pre-existing docker volume for the graph db.") @click.option('--no-redis', is_flag=True, help='Only allowed when RedisGraph is sole GDB, which is then used in place of Redis.') def generate(gdb, pfsc_tag, frontend_tag, oca_tag, official, workers, demos, mount_code, mount_pkg, dump_dc, dirname, no_local, flask_config, per_deploy_dirs, static_redir, static_acao, dummy, @@ -742,10 +742,13 @@ def write_docker_compose_yaml(deploy_dir_name, deploy_dir_path, gdb, pfsc_tag, f svc_defn = writer(altdir=altdir) if gdb_vol: - # TODO: Provide support for use of named volumes with other GDBs besides RedisGraph - if code == GdbCode.RE: + data_vol_mount_pt = GdbCode.service_container_data_vol_mount_point(code) + if data_vol_mount_pt is None: + name = GdbCode.service_name(code) + raise click.UsageError(f'--gdb-vol switch is not currently supported for {name}') + else: svc_defn['volumes'] = [ - f'{gdb_vol}:/data' + f'{gdb_vol}:{data_vol_mount_pt}' ] s_full[name] = svc_defn @@ -768,7 +771,7 @@ def write_pfsc_service(cmd): return services.pise_server(deploy_dir_path, cmd, flask_config, gdb, tag=pfsc_tag, workers=workers, demos=demos, mount_code=mount_code, mount_pkg=mount_pkg, official=official, altdir=altdir, - lib_vol=lib_vol, build_vol=build_vol, no_redis=no_redis) + lib_vol=lib_vol, build_vol=build_vol, gdb_vol=gdb_vol, no_redis=no_redis) for n in range(workers): svc_pfscwork = write_pfsc_service('worker') diff --git a/manage/tools/deploy/services.py b/manage/tools/deploy/services.py index b0f3196..a1abf60 100644 --- a/manage/tools/deploy/services.py +++ b/manage/tools/deploy/services.py @@ -120,6 +120,15 @@ def service_defn_writer(cls, code): cls.JA: janusgraph, }[code] + @classmethod + def service_container_data_vol_mount_point(cls, code): + """ + Give the mount point within the service container where the GDB stores its data. + """ + return { + cls.RE: '/data', + }.get(code, None) + @classmethod def GremLite_URI(cls, pfsc_root_path): db_file_path = pathlib.Path(pfsc_root_path) / 'graphdb/gl/gremlite.db' @@ -261,7 +270,7 @@ def pise_server(deploy_dir_path, mode, flask_config, gdb, tag='latest', workers=1, demos=False, mount_code=False, mount_pkg=None, official=False, altdir=None, - lib_vol=None, build_vol=None, + lib_vol=None, build_vol=None, gdb_vol=None, no_redis=False): d = { 'image': f"{'proofscape/' if official else ''}pise-server:{tag}", @@ -303,23 +312,36 @@ def pise_server(deploy_dir_path, mode, flask_config, gdb, tag='latest', }[mode] d['environment'][mode_env_var] = 1 + # Both web server and workers should wait for any GDB services to be ready. + d['depends_on'].extend(GdbCode.service_name(code) for code in gdb if code in GdbCode.via_container) + + # Web server depends on workers. if mode == 'websrv': - d['depends_on'].extend(GdbCode.service_name(code) for code in gdb if code in GdbCode.via_container) d['depends_on'].extend([f'pfscwork{n}' for n in range(workers)]) + + # Both web server and all workers should have access to the same GremLite db file, if using GremLite. if GdbCode.GL in gdb: direc = 'graphdb/gl' - d['volumes'].append(f'{get_proofscape_subdir_abs_fs_path_on_host(direc, altdir=altdir)}:/proofscape/{direc}') + # It is only for this use case that this function accepts a `gdb_vol` kwarg. + volume = gdb_vol or get_proofscape_subdir_abs_fs_path_on_host(direc, altdir=altdir) + d['volumes'].append(f'{volume}:/proofscape/{direc}') + if conf.EMAIL_TEMPLATE_DIR: d['volumes'].append(f'{resolve_fs_path("EMAIL_TEMPLATE_DIR")}:/home/pfsc/proofscape/src/_email_templates:ro') + + # If mounting code, it goes into all web server and workers if mount_code: if demos: d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-demo-repos")}:/home/pfsc/demos:ro') d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-server/pfsc")}:/home/pfsc/proofscape/src/pfsc-server/pfsc:ro') d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-server/config.py")}:/home/pfsc/proofscape/src/pfsc-server/config.py:ro') d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-ise/package.json")}:/home/pfsc/proofscape/src/client/package.json:ro') + + # Likewise for packages if mount_pkg: for pkg in [s.strip() for s in mount_pkg.split(',')]: d['volumes'].append(f'{resolve_pfsc_root_subdir("src/pfsc-server/venv/lib/python3.8/site-packages")}/{pkg}:/usr/local/lib/python3.8/site-packages/{pkg}') + return d From f9b1681a24a8005093ffba86d7b177fd94fe6537 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 16:08:13 -0500 Subject: [PATCH 21/35] Adjust build-and-test workflow to use GremLite with both MCA and OCA --- .github/workflows/pise-build-and-test.yml | 25 ++++------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index 9f54a48..37ae1ef 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -311,26 +311,9 @@ jobs: echo "PFSC_PDF_VERS=`pfsc check version pfsc-pdf`" >> $GITHUB_OUTPUT echo "PFSC_EXAMP_VERS=`pfsc check version pfsc-examp`" >> $GITHUB_OUTPUT echo "PYODIDE_VERS=`pfsc check version pyodide`" >> $GITHUB_OUTPUT - echo "REDISGRAPH_VERS=`pfsc check version redisgraph-tag`" >> $GITHUB_OUTPUT echo "NGINX_VERS=`pfsc check version nginx-tag`" >> $GITHUB_OUTPUT echo "DEMO_REPO_VERS=`pfsc check version demo-repos`" >> $GITHUB_OUTPUT # ----------------------------------------- - # Obtain RedisGraph image, preferably from cache - - name: Cache redisgraph - id: cache-redisgraph - uses: actions/cache@v4 - with: - path: images/redisgraph.tar - key: cache-redisgraph-${{ steps.vers-nums.outputs.REDISGRAPH_VERS }} - - if: ${{ steps.cache-redisgraph.outputs.cache-hit == 'true' }} - name: Load redisgraph - run: docker load --input=images/redisgraph.tar - - if: ${{ steps.cache-redisgraph.outputs.cache-hit != 'true' }} - name: Pull redisgraph - run: | - docker pull redis/redis-stack-server:${{ steps.vers-nums.outputs.REDISGRAPH_VERS }} - docker save --output=images/redisgraph.tar redis/redis-stack-server:${{ steps.vers-nums.outputs.REDISGRAPH_VERS }} - # ----------------------------------------- # Obtain Nginx image, preferably from cache - name: Cache nginx id: cache-nginx @@ -452,8 +435,8 @@ jobs: working-directory: pise/manage run: | source venv/bin/activate - pfsc deploy generate --gdb re \ - ${{ !inputs.pub-prep && '--no-redis --mount-code' || '--no-mount-code' }} \ + pfsc deploy generate --gdb gl \ + ${{ !inputs.pub-prep && '--mount-code' || '--no-mount-code' }} \ --pfsc-tag ${{ env.PISE_VERS }} \ --oca-tag ${{ env.PISE_VERS }} \ -n 1 --demos \ @@ -777,8 +760,8 @@ jobs: docker logs ${{env.TEST_DEPLOY_DIR}}-pise-1 > selenium_results/docker_logs/pise-1.txt 2>&1 docker exec ${{env.TEST_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/pfsc_web-stdout*.log" > selenium_results/docker_logs/pise-1-web-stdout.txt 2>&1 docker exec ${{env.TEST_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/pfsc_web-stderr*.log" > selenium_results/docker_logs/pise-1-web-stderr.txt 2>&1 - docker exec ${{env.TEST_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/redisgraph-stdout*.log" > selenium_results/docker_logs/pise-1-rg-stdout.txt 2>&1 - docker exec ${{env.TEST_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/redisgraph-stderr*.log" > selenium_results/docker_logs/pise-1-rg-stderr.txt 2>&1 + docker exec ${{env.TEST_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/redis-stdout*.log" > selenium_results/docker_logs/pise-1-redis-stdout.txt 2>&1 + docker exec ${{env.TEST_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/redis-stderr*.log" > selenium_results/docker_logs/pise-1-redis-stderr.txt 2>&1 - if: ${{ inputs.pub-prep && always() }} name: Upload selenium results uses: actions/upload-artifact@v4 From 296a763a0abd6c5f539fd69fb21ddc67e57300e0 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Tue, 10 Feb 2026 16:08:28 -0500 Subject: [PATCH 22/35] Incidental improvements to build-and-test workflow --- .github/workflows/pise-build-and-test.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index 37ae1ef..5c1e6b2 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -107,7 +107,7 @@ jobs: uses: actions/cache@v4 with: path: pise/server/venv - key: cache-server-venv-${{ hashFiles('pise/server/req/*') }} + key: cache-server-venv-${{ hashFiles('pise/server/req/*', 'pise/manage/pise_python_vers.env') }} - if: ${{ steps.cache-server-venv.outputs.cache-hit != 'true' }} name: Install server/venv working-directory: pise/server @@ -123,17 +123,17 @@ jobs: git config --global user.email "pfsc.unit.tester@localhost" git config --global user.name "pfsc unit tester" source venv/bin/activate - python -m tests.util.make_repos + inv mtr - name: Build test repos working-directory: pise/server run : | source venv/bin/activate - python -m tests.util.build_repos + inv btr - name: Run unit tests working-directory: pise/server run: | source venv/bin/activate - pytest tests + inv unit # ============================================================================= builds-and-functional-tests: needs: server-unit-tests @@ -207,7 +207,7 @@ jobs: uses: actions/cache@v4 with: path: pise/manage/venv - key: cache-manage-venv-${{ hashFiles('pise/manage/setup.py') }} + key: cache-manage-venv-${{ hashFiles('pise/manage/setup.cfg', 'pise/manage/pise_python_vers.env') }} - if: ${{ steps.cache-manage-venv.outputs.cache-hit != 'true' }} name: Form the manage/venv directory working-directory: pise/manage @@ -241,7 +241,7 @@ jobs: uses: actions/cache@v4 with: path: pise/server/venv - key: cache-server-venv-${{ hashFiles('pise/server/req/*') }} + key: cache-server-venv-${{ hashFiles('pise/server/req/*', 'pise/manage/pise_python_vers.env') }} - if: ${{ steps.cache-server-venv.outputs.cache-hit != 'true' }} name: Install server/venv working-directory: pise/server @@ -473,7 +473,7 @@ jobs: git config --global user.email "pfsc.test.runner@localhost" git config --global user.name "pfsc test runner" source venv/bin/activate - python -m tests.util.make_repos + inv mtr # List test repo owners - if: ${{ env.DEBUG_WORKFLOW >= 1 }} name: List test repo owners @@ -711,7 +711,8 @@ jobs: test `grep NOTICE.txt $WDLL | awk '{print $5}'` -ge 1000 test `grep about.json $WDLL | awk '{print $5}'` -ge 20000 # ----------------------------------------- - # Remake the lib, build, gdb volumes + # We remake the lib, build, gdb volumes fresh for use with the OCA. + # First remove the existing ones - if: ${{ inputs.pub-prep }} name: Remove existing lib, build, gdb volumes run: docker volume rm ${{env.LIB_VOLUME}} ${{env.BUILD_VOLUME}} ${{env.GDB_VOLUME}} From 5e2056fd8bfb9e1063722ce2f0a3e791e399c872 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Wed, 11 Feb 2026 09:44:03 -0500 Subject: [PATCH 23/35] Programmatically determine service versions in build-and-test wkflw --- .github/workflows/pise-build-and-test.yml | 63 +++++++++++++++++++++-- manage/tools/util.py | 1 + 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index 5c1e6b2..ddece97 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -40,19 +40,72 @@ on: default: 1 required: false jobs: + # ============================================================================= + # This job determines the version numbers of the service containers that are + # needed for the next job, which runs the pise/server unit tests. + set-svc-versions: + runs-on: ubuntu-22.04 + steps: + - name: Checkout pise + uses: actions/checkout@v4 + with: + path: 'pise' + - name: Determine Python version + working-directory: pise/manage + run: | + . pise_python_vers.env + echo "PISE_PYTHON_VERS=$PISE_PYTHON_VERS" >> $GITHUB_ENV + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PISE_PYTHON_VERS }} + # ----------------------------------------- + # Install pise/manage + - name: Cache manage/venv + id: cache-manage-venv + uses: actions/cache@v4 + with: + path: pise/manage/venv + key: cache-manage-venv-${{ hashFiles('pise/manage/setup.cfg', 'pise/manage/pise_python_vers.env') }} + - if: ${{ steps.cache-manage-venv.outputs.cache-hit != 'true' }} + name: Initialize manage/venv directory + working-directory: pise/manage + run: | + python -m venv venv + - name: Install pfsc-manage + working-directory: pise/manage + run: | + source venv/bin/activate + pip install -e . + # ----------------------------------------- + - name: Determine version numbers + id: vers-nums + working-directory: pise/manage + run: | + source venv/bin/activate + echo "redis_tag=`pfsc check version redis-tag`" >> $GITHUB_OUTPUT + echo "redisgraph_tag=`pfsc check version redisgraph-tag`" >> $GITHUB_OUTPUT + echo "tinkergraph_tag=`pfsc check version tinkergraph-tag`" >> $GITHUB_OUTPUT + outputs: + redis_tag: ${{ steps.vers-nums.outputs.redis_tag }} + redisgraph_tag: ${{ steps.vers-nums.outputs.redisgraph_tag }} + tinkergraph_tag: ${{ steps.vers-nums.outputs.tinkergraph_tag }} + # ============================================================================= + # This job runs the pise/server unit tests, against several graph databases. server-unit-tests: + needs: set-svc-versions runs-on: ubuntu-22.04 services: redis: - image: "redis:8.4.0-bookworm" + image: ${{ format('redis:{0}', needs.set-svc-versions.outputs.redis_tag) }} ports: - "6379:6379" redisgraph: - image: "redis/redis-stack-server:6.2.6-v6" + image: ${{ format('redis/redis-stack-server:{0}', needs.set-svc-versions.outputs.redisgraph_tag) }} ports: - "6381:6379" tinkergraph: - image: "tinkerpop/gremlin-server:3.7.5" + image: ${{ format('tinkerpop/gremlin-server:{0}', needs.set-svc-versions.outputs.tinkergraph_tag) }} ports: - "8182:8182" strategy: @@ -134,7 +187,9 @@ jobs: run: | source venv/bin/activate inv unit -# ============================================================================= + # ============================================================================= + # This job builds docker images and the client-side code, does functional tests + # of PISE, and, in the case of a pub-prep run, uploads the products as artifacts. builds-and-functional-tests: needs: server-unit-tests runs-on: ubuntu-22.04 diff --git a/manage/tools/util.py b/manage/tools/util.py index 82769b1..792027f 100644 --- a/manage/tools/util.py +++ b/manage/tools/util.py @@ -156,6 +156,7 @@ def get_version_numbers(include_tags=False, include_other=False): # Could add others; atm this is all we need nums['redis-tag'] = pfsc_conf.REDIS_IMAGE_TAG nums['redisgraph-tag'] = pfsc_conf.REDISGRAPH_IMAGE_TAG + nums['tinkergraph-tag'] = pfsc_conf.GREMLIN_SERVER_IMAGE_TAG nums['nginx-tag'] = pfsc_conf.NGINX_IMAGE_TAG if include_other: From efb4e80145082f4fc3bdbc54a37ebfea95b672a2 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Wed, 11 Feb 2026 11:22:47 -0500 Subject: [PATCH 24/35] Fix workspace interp. in gremlite URI --- .github/workflows/pise-build-and-test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index ddece97..fb57195 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -112,7 +112,9 @@ jobs: matrix: include: - name: GremLite - graphdb_uri: ${{ format('file://{0}/graphdb/gl/gremlite.db', github.workspace) }} + # Note: Cannot interpolate `github.workspace` into the `{0}` here because apparently + # it is not defined yet. (Returns empty string.) So we interpolate it below instead. + graphdb_uri: 'file://{0}/graphdb/gl/gremlite.db' redis_uri: redis://localhost:6379 - name: RedisGraph graphdb_uri: redis://localhost:6381 @@ -125,7 +127,11 @@ jobs: shell: bash env: REDIS_URI: ${{ matrix.redis_uri }} - GRAPHDB_URI: ${{ matrix.graphdb_uri }} + GRAPHDB_URI: ${{ + startsWith(matrix.graphdb_uri, 'file://') + && format(matrix.graphdb_uri, github.workspace) + || matrix.graphdb_uri + }} PFSC_LIB_ROOT: ${{ format('{0}/lib', github.workspace) }} PFSC_BUILD_ROOT: ${{ format('{0}/build', github.workspace) }} SECRET_KEY: fixed_value_for_testing From 01a905db99ef14b97a83338aecd6d574922426fd Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Wed, 11 Feb 2026 12:29:36 -0500 Subject: [PATCH 25/35] Improve cache key for pise-server image --- .github/workflows/pise-build-and-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index fb57195..c1873a3 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -553,7 +553,9 @@ jobs: uses: actions/cache@v4 with: path: images/pise-server.tar - key: cache-pise-server-${{env.PISE_VERS}} + key: cache-pise-server-${{env.PISE_VERS}}-${{ + hashFiles('pise/server/req/*', 'pise/manage/pise_python_vers.env') + }} # If basic testing and had a cache hit, load from tar file into Docker. - if: ${{!inputs.pub-prep && steps.cache-pise-server.outputs.cache-hit == 'true' }} name: Load pise-server From 8c557176c8454c7814c8a0535f4167c6727d21e7 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 20 Feb 2026 13:59:02 -0500 Subject: [PATCH 26/35] Move to gremlite v0.37.0-rc1 --- server/req/requirements.in | 2 +- server/req/requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/req/requirements.in b/server/req/requirements.in index d8db83c..73274b7 100644 --- a/server/req/requirements.in +++ b/server/req/requirements.in @@ -26,7 +26,7 @@ neo4j==4.4.13 redisgraph==2.4.4 gremlinpython==3.7.5 wsc-grempy-transport==0.1.0 -gremlite==0.1.1 +gremlite==0.37.0rc1 # Lark parses all of our basic languages, such as the pfsc module syntax, # and the Meson proof script language. diff --git a/server/req/requirements.txt b/server/req/requirements.txt index 184961b..51ce56b 100644 --- a/server/req/requirements.txt +++ b/server/req/requirements.txt @@ -615,9 +615,9 @@ gremlinpython==3.7.5 \ # -r requirements.in # gremlite # wsc-grempy-transport -gremlite==0.1.1 \ - --hash=sha256:111b71f0528b41a660d8da66f1367ae39d962094285481a240ba7604f8a8bcd5 \ - --hash=sha256:a0e1173a3170a809a3dc33aa0a82064055552d3f573c4361fa235757cb730982 +gremlite==0.37.0rc1 \ + --hash=sha256:324954e38831caa529959e63df16c0d61b5aaa134eaf1c80ff5e9ba670e290ab \ + --hash=sha256:8a373babedf6cb78408a5cdc5442ae446cdc823e6d527283b4c513d2bce6488b # via -r requirements.in h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ From bf4bb4c77de3de1bfcd7e773baef6055f69f9cf7 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 20 Feb 2026 13:59:27 -0500 Subject: [PATCH 27/35] Add diagnostics for build issues --- .github/workflows/pise-build-and-test.yml | 2 +- server/tests/util/build_repos.py | 29 ++++++++++++++++------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index c1873a3..ab26f10 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -187,7 +187,7 @@ jobs: working-directory: pise/server run : | source venv/bin/activate - inv btr + time inv btr - name: Run unit tests working-directory: pise/server run: | diff --git a/server/tests/util/build_repos.py b/server/tests/util/build_repos.py index 288e3b7..4e5d7ae 100644 --- a/server/tests/util/build_repos.py +++ b/server/tests/util/build_repos.py @@ -18,17 +18,30 @@ Build and index the test repos. """ +import sqlite3 + from tests.util import build_all, build_at_wip, build_big +from gremlite.logging import print_open_cursor_traces + if __name__ == "__main__": - # Uncomment to record logs: - #import logging - #logging.basicConfig(filename='build_repos.log', level=logging.INFO) - #build_big() + # The try-except was added in order to help diagnose issues with transactions + # when first adding support for gremlite. I leave it in case it is useful for + # future debugging. + try: + # Uncomment to record logs: + #import logging + #logging.basicConfig(filename='build_repos.log', level=logging.INFO) + + #build_big() + + build_all() + # To show timings: + #build_all(verbose=2) - build_all() - # To show timings: - #build_all(verbose=2) + build_at_wip() - build_at_wip() + except sqlite3.OperationalError: + print_open_cursor_traces() + raise From 1911bf572615adb9e8da9df542243152557d54a9 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 20 Feb 2026 14:04:09 -0500 Subject: [PATCH 28/35] Make gremlin reader use existing transaction when writer has one This makes it so the reader remains usable by the writer, while the latter is in the middle of a transaction and has already made changes. Otherwise, the reader may timeout and raise an sqlite "database locked" exception. --- server/pfsc/gdb/gremlin/reader.py | 4 +++- server/pfsc/gdb/reader.py | 8 ++++++++ server/pfsc/gdb/writer.py | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/server/pfsc/gdb/gremlin/reader.py b/server/pfsc/gdb/gremlin/reader.py index bd34f00..ccf5439 100644 --- a/server/pfsc/gdb/gremlin/reader.py +++ b/server/pfsc/gdb/gremlin/reader.py @@ -32,7 +32,9 @@ class GremlinGraphReader(GraphReader): @property def g(self): - return self.gdb + # We use `writer.g` if possible, since that might be a transaction, + # in which case we should participate in it. + return self.writer.g if self.writer is not None else self.gdb def num_nodes_in_db(self): return self.g.V().count().next() diff --git a/server/pfsc/gdb/reader.py b/server/pfsc/gdb/reader.py index d5129c5..a4806bd 100644 --- a/server/pfsc/gdb/reader.py +++ b/server/pfsc/gdb/reader.py @@ -80,6 +80,14 @@ class GraphReader: def __init__(self, gdb): self.gdb = gdb + self._writer = None + + def set_writer(self, w): + self._writer = w + + @property + def writer(self): + return self._writer @staticmethod def adaptall(version): diff --git a/server/pfsc/gdb/writer.py b/server/pfsc/gdb/writer.py index d7f5676..608068b 100644 --- a/server/pfsc/gdb/writer.py +++ b/server/pfsc/gdb/writer.py @@ -38,9 +38,10 @@ def wrapper(graph_writer, *args, **kwargs): class GraphWriter: """Abstract base class for graph database writers. """ - def __init__(self, reader): + def __init__(self, reader: GraphReader): self.gdb = reader.gdb self._reader = reader + reader.set_writer(self) self._tx = None @property From 6d7cffb19aabd0a1e66c4dc6ebd880eab43d2b27 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Sat, 21 Feb 2026 06:40:01 -0500 Subject: [PATCH 29/35] Form graphdb subdirs in pise-server dockerfiles --- manage/topics/pfsc/templates/Dockerfile.pfsc | 7 ++++++- manage/topics/pfsc/templates/Dockerfile.startup_system | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/manage/topics/pfsc/templates/Dockerfile.pfsc b/manage/topics/pfsc/templates/Dockerfile.pfsc index ea85c30..177a767 100644 --- a/manage/topics/pfsc/templates/Dockerfile.pfsc +++ b/manage/topics/pfsc/templates/Dockerfile.pfsc @@ -17,10 +17,15 @@ {# Make `pfsc` user, `/home/pfsc/proofscape` dir, etc. #} {% if ubuntu %} +# Note: We form graphdb subdirectories like graphdb/gl etc. now so that they +# have the right ownership in the container. If they are instead formed by docker when +# the container is started up (due to volume mount), then they will be owned by root, +# and the pfsc user cannot write to them. RUN adduser --disabled-password --gecos "" pfsc \ && sed -i -e "s/#alias ll=/alias ll=/" /home/pfsc/.bashrc \ && sed -i -e "s%# \(export\|eval\|alias l\)%\1%" /root/.bashrc \ - && bash -c "mkdir -p /proofscape/{lib,build/_sphinx,graphdb,deploy,PDFLibrary}" + && bash -c "mkdir -p /proofscape/{lib,build/_sphinx,graphdb,deploy,PDFLibrary}" \ + && bash -c "mkdir -p /proofscape/graphdb/{gl,re}" {% else %} RUN adduser -D pfsc \ && mkdir /proofscape diff --git a/manage/topics/pfsc/templates/Dockerfile.startup_system b/manage/topics/pfsc/templates/Dockerfile.startup_system index 8ced866..8e1fc42 100644 --- a/manage/topics/pfsc/templates/Dockerfile.startup_system +++ b/manage/topics/pfsc/templates/Dockerfile.startup_system @@ -36,10 +36,15 @@ COPY {{tmp_dir_name}}/{{name}}.ini super/{{num}}_{{name}}.ini RUN chown -R pfsc:pfsc super # `-n` switch runs supervisord in the foreground. See . +# Note: We form graphdb subdirectories like graphdb/gl etc. now so that they +# have the right ownership in the container. If they are instead formed by docker when +# the container is started up (due to volume mount), then they will be owned by root, +# and the pfsc user cannot write to them. ARG STARTUP=/usr/local/bin/startup RUN echo "#!/bin/bash" > $STARTUP \ {% if ensure_dirs %} \ && echo "mkdir -p /proofscape/{lib,build,graphdb,deploy,PDFLibrary}" >> $STARTUP \ + && echo "mkdir -p /proofscape/graphdb/{gl,re}" >> $STARTUP \ {% endif %} \ && echo "exec supervisord -n -c {{dir_where_startup_system_lives}}/super/supervisord.conf" >> $STARTUP \ && chmod +x $STARTUP From 70e7b39a38356f652c37b5965175399826b2be1d Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Sat, 21 Feb 2026 06:40:18 -0500 Subject: [PATCH 30/35] Include dockerfile templates dir in pise-server cache key hash --- .github/workflows/pise-build-and-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index ab26f10..32b7186 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -554,7 +554,7 @@ jobs: with: path: images/pise-server.tar key: cache-pise-server-${{env.PISE_VERS}}-${{ - hashFiles('pise/server/req/*', 'pise/manage/pise_python_vers.env') + hashFiles('pise/server/req/*', 'pise/manage/pise_python_vers.env', 'pise/manage/topics/pfsc/templates') }} # If basic testing and had a cache hit, load from tar file into Docker. - if: ${{!inputs.pub-prep && steps.cache-pise-server.outputs.cache-hit == 'true' }} From 08767f31cbd973722bdeab3046a09ac0a884c672 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Sat, 21 Feb 2026 10:40:09 -0500 Subject: [PATCH 31/35] Support building two versions (-gl and -rg) of the OCA --- changelog.d/99.changed.txt | 4 --- changelog.d/99.improved.txt | 10 ++++++ manage/tools/build.py | 33 +++++++++++++++---- manage/tools/license.py | 3 +- manage/tools/util.py | 2 +- manage/topics/pfsc/__init__.py | 13 +++++--- manage/topics/pfsc/templates/Dockerfile.oca | 28 ++++++++++++++++ .../pfsc/templates/Dockerfile.oca_final_setup | 22 +++++++++++++ server/config.py | 6 +++- 9 files changed, 104 insertions(+), 17 deletions(-) delete mode 100644 changelog.d/99.changed.txt diff --git a/changelog.d/99.changed.txt b/changelog.d/99.changed.txt deleted file mode 100644 index 4bcc36e..0000000 --- a/changelog.d/99.changed.txt +++ /dev/null @@ -1,4 +0,0 @@ -The [PISE one-container app Docker image](https://hub.docker.com/r/proofscape/pise) now -uses `GremLite` instead of `RedisGraph` as its built-in graph database system. -Users of previous versions of the app will have to rebuild all Proofscape repos in their -library, as the new app will not be able to read the old graph database files. diff --git a/changelog.d/99.improved.txt b/changelog.d/99.improved.txt index 522dab8..3b88c6a 100644 --- a/changelog.d/99.improved.txt +++ b/changelog.d/99.improved.txt @@ -1,2 +1,12 @@ `pise/server` now supports the [GremLite](https://pypi.org/project/gremlite/) graph database system. + +FOR NOW: Two versions of the +[PISE one-container app Docker image](https://hub.docker.com/r/proofscape/pise) +will be offered, one using RedisGraph (tags ending in `-rg`), and one using +GremLite (tags ending in `-gl`). + +FUTURE: The RedisGraph versions of the Docker image will eventually be dropped, +and only GremLite versions will be produced. Users of RedisGraph versions of the +app will have to rebuild all Proofscape repos in their library, as the GremLite +versions will not be able to read the old graph database files. diff --git a/manage/tools/build.py b/manage/tools/build.py index 771947d..e2ab858 100644 --- a/manage/tools/build.py +++ b/manage/tools/build.py @@ -27,6 +27,7 @@ import conf from manage import cli, PFSC_ROOT, PFSC_MANAGE_ROOT from conf import DOCKER_CMD, DOCKER_PLATFORM +from tools.deploy.services import GdbCode import tools.license from tools.util import get_version_numbers import topics.pfsc.write_license_files as write_license_files @@ -280,12 +281,15 @@ def oca_readiness_checks(release=False, client=True, client_min=False, pdf=True, @build.command() +@click.option('--gdb', + default=GdbCode.GL, prompt='Graph database (gl, re)', + help='Choose the graph DB to be built into the image. gl=GremLite, re=RedisGraph') @click.option('--dump', is_flag=True, help="Dump Dockerfile to stdout before building.") @click.option('--dry-run', is_flag=True, help="Do not actually build; just print docker command.") @click.option('--tar-path', help="Instead of building, save the context tar file to this path.") @click.option('--skip-licensing', is_flag=True, default=False, help='Skip gathering of license info. For dev only.') @click.argument('tag') -def oca(dump, dry_run, tar_path, skip_licensing, tag: str): +def oca(gdb, dump, dry_run, tar_path, skip_licensing, tag: str): """ Build a `pise` (one-container app) docker image, and give it a TAG. @@ -294,6 +298,14 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str): content repos. It is configured in "personal server mode" and comes with RedisGraph as the GDB. """ + if gdb not in [GdbCode.GL, GdbCode.RE]: + raise click.UsageError('Must select one of the graph databases GL (GremLite) or RE (RedisGraph).') + + gdb_tag_segment_lookup = {GdbCode.GL: 'gl', GdbCode.RE: 'rg'} + segment = gdb_tag_segment_lookup[gdb] + if segment not in tag.split('-'): + raise click.UsageError(f'When using the {gdb} graph database, the tag must contain "-{segment}".') + if not dry_run: oca_readiness_checks(client=True, client_min=False, pdf=True, pyodide=True, whl=True) @@ -307,7 +319,7 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str): from topics.pfsc import write_oca_eula_file from topics.pfsc import write_worker_and_web_supervisor_ini from topics.pfsc import write_proofscape_oca_dockerfile - from topics.redis import write_redis_ini + from topics.redis import write_redis_ini, write_redisgraph_ini with tempfile.TemporaryDirectory(dir=SRC_TMP_ROOT) as tmp_dir_name: with open(os.path.join(tmp_dir_name, 'license_info.json'), 'w') as f: f.write(json.dumps(license_info, indent=4)) @@ -322,14 +334,23 @@ def oca(dump, dry_run, tar_path, skip_licensing, tag: str): ini = write_worker_and_web_supervisor_ini( worker=False, web=True, use_venv=False, oca=True) f.write(ini) - with open(os.path.join(tmp_dir_name, 'redis.ini'), 'w') as f: - ini = write_redis_ini(use_conf_file=True) - f.write(ini) + + if gdb == GdbCode.GL: + with open(os.path.join(tmp_dir_name, 'redis.ini'), 'w') as f: + ini = write_redis_ini(use_conf_file=True) + f.write(ini) + else: + assert gdb == GdbCode.RE + with open(os.path.join(tmp_dir_name, 'redisgraph.ini'), 'w') as f: + ini = write_redisgraph_ini(use_conf_file=True) + f.write(ini) + with open(os.path.join(tmp_dir_name, 'oca_version.txt'), 'w') as f: f.write(tag) + tmp_dir_rel_path = os.path.relpath(tmp_dir_name, start=SRC_ROOT) write_dockerignore_for_pyc() - df = write_proofscape_oca_dockerfile(tmp_dir_rel_path) + df = write_proofscape_oca_dockerfile(gdb, tmp_dir_rel_path) finalize(df, 'pise', tag, dump, dry_run, tar_path=tar_path) diff --git a/manage/tools/license.py b/manage/tools/license.py index 002e972..45165aa 100644 --- a/manage/tools/license.py +++ b/manage/tools/license.py @@ -296,7 +296,8 @@ def gather_licensing_info(verbose=False): vers['python'] = get_python_version_for_images() vers['supervisor'] = pfsc_conf.SUPERVISOR_VERSION vers['redisgraph'] = pfsc_conf.REDISGRAPH_IMAGE_TAG - vers['redis-server'] = get_redis_server_version_for_oca() + vers['redis-server-rg-oca'] = pfsc_conf.REDISGRAPH_IMAGE_TAG.split('-')[0] + vers['redis-server-gl-oca'] = get_redis_server_version_for_oca() vers['nginx'] = pfsc_conf.NGINX_IMAGE_TAG # These are the one-off cases: diff --git a/manage/tools/util.py b/manage/tools/util.py index 792027f..cda00cf 100644 --- a/manage/tools/util.py +++ b/manage/tools/util.py @@ -179,7 +179,7 @@ def get_server_version(): def get_redis_server_version_for_oca(): """ - Get the version number of redis-server that is installed in the OCA image. + Get the version number of redis-server that is installed in the GremLite OCA image. """ cmd = f'docker run --rm --entrypoint=bash redis:{pfsc_conf.REDIS_IMAGE_TAG} -c "redis-server --version"' out = subprocess.check_output(cmd, shell=True) diff --git a/manage/topics/pfsc/__init__.py b/manage/topics/pfsc/__init__.py index 12a6221..4bf990d 100644 --- a/manage/topics/pfsc/__init__.py +++ b/manage/topics/pfsc/__init__.py @@ -24,6 +24,7 @@ from manage import PFSC_ROOT from tools.util import squash from tools.deploy import list_wheel_filenames +from tools.deploy.services import GdbCode from tools.util import get_version_numbers, get_server_version @@ -129,10 +130,11 @@ def write_oca_static_setup(tmp_dir_name): ) -def write_oca_final_setup(tmp_dir_name, final_workdir='/home/pfsc'): +def write_oca_final_setup(gdb, tmp_dir_name, final_workdir='/home/pfsc'): template = jinja_env.get_template('Dockerfile.oca_final_setup') versions = get_version_numbers() return template.render( + gdb=gdb, tmp_dir_name=tmp_dir_name, final_workdir=final_workdir, ) @@ -192,7 +194,8 @@ def write_frontend_dockerfile(tmp_dir_name): return squash(df) -def write_proofscape_oca_dockerfile(tmp_dir_name, demos=False): +def write_proofscape_oca_dockerfile(gdb, tmp_dir_name, demos=False): + assert gdb in [GdbCode.GL, GdbCode.RE] pfsc_install = write_pfsc_installation( ubuntu=True, demos=demos, use_venv=False, oca_version_file=f'{tmp_dir_name}/oca_version.txt', @@ -200,7 +203,7 @@ def write_proofscape_oca_dockerfile(tmp_dir_name, demos=False): ) startup_system = write_startup_system( '/home/pfsc', numbered_inis={ - 100: 'redis', + 100: 'redisgraph' if gdb == GdbCode.RE else 'redis', 200: 'pfsc', }, tmp_dir_name=tmp_dir_name ) @@ -208,12 +211,14 @@ def write_proofscape_oca_dockerfile(tmp_dir_name, demos=False): tmp_dir_name ) final_setup = write_oca_final_setup( - tmp_dir_name, final_workdir='/home/pfsc' + gdb, tmp_dir_name, final_workdir='/home/pfsc' ) template = jinja_env.get_template('Dockerfile.oca') df = template.render( + gdb=gdb, python_image_tag=conf.PYTHON_IMAGE_TAG, redis_image_tag=conf.REDIS_IMAGE_TAG, + redisgraph_image_tag=conf.REDISGRAPH_IMAGE_TAG, platform=conf.DOCKER_PLATFORM, pfsc_install=pfsc_install, startup_system=startup_system, diff --git a/manage/topics/pfsc/templates/Dockerfile.oca b/manage/topics/pfsc/templates/Dockerfile.oca index 08c4fce..7091d9b 100644 --- a/manage/topics/pfsc/templates/Dockerfile.oca +++ b/manage/topics/pfsc/templates/Dockerfile.oca @@ -14,6 +14,32 @@ # limitations under the License. # # --------------------------------------------------------------------------- # +{% if gdb=="re" %} + +FROM --platform={{platform}} redis/redis-stack-server:{{redisgraph_image_tag}} AS rg +FROM --platform={{platform}} python:3.11.13-slim-bullseye AS basis +ARG DEBIAN_FRONTEND=noninteractive +# `libgomp1` is needed by redisgraph. +# `sudo` and `less` are installed for dev purposes, and user `pfsc` (though it +# isn't added until later in the Dockerfile) is given passwordless sudo. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libgomp1 sudo less \ + && rm -rf /var/lib/apt/lists/* \ + && echo "pfsc ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && mkdir -p /usr/lib/redis/modules +COPY --from=rg /opt/redis-stack/lib/redisgraph.so /usr/lib/redis/modules +# We don't need redisearch.so for our purposes; we load it only so that we +# can successfully read RDB files that were dumped by a Redis that does have +# redisearch.so. See +# The latter occurs e.g. when the dump file was created by pise/server unit +# tests, running against a redis/redis-stack-server image. +COPY --from=rg /opt/redis-stack/lib/redisearch.so /usr/lib/redis/modules +COPY --from=rg /usr/bin/redis-server /usr/local/bin +# Note: Could save 6MB by ignoring redis-cli, which is just a debugging tool. +COPY --from=rg /usr/bin/redis-cli /usr/local/bin + +{% else %} + FROM --platform={{platform}} redis:{{redis_image_tag}} AS redis_image FROM --platform={{platform}} python:{{python_image_tag}} AS basis ARG DEBIAN_FRONTEND=noninteractive @@ -25,6 +51,8 @@ RUN apt-get update \ && echo "pfsc ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers COPY --from=redis_image /usr/local/bin/redis-server /usr/local/bin +{% endif %} + # For the sake of casual users, I think it's important that the Proofscape ISE # app port 7372 get top billing in Docker Dashboard, in particular in the # "Run" dialog. Therefore, we don't expose 6379. Casual users shouldn't have diff --git a/manage/topics/pfsc/templates/Dockerfile.oca_final_setup b/manage/topics/pfsc/templates/Dockerfile.oca_final_setup index 3718a7c..ba023b9 100644 --- a/manage/topics/pfsc/templates/Dockerfile.oca_final_setup +++ b/manage/topics/pfsc/templates/Dockerfile.oca_final_setup @@ -21,11 +21,33 @@ USER pfsc WORKDIR {{final_workdir}} # redis.conf +{% if gdb=="re" %} + +# In particular, `save 1 1` means that, when there are any changes in our GDB +# (RedisGraph), it will be at most 1 second before Redis initiates a BGSAVE to +# commit them to disk. Combined with use of `pfsc.gdb.cypher.rg.redis_bg_save()` +# in pfsc-server, this should achieve pretty robust persistence. +RUN echo "loadmodule /usr/lib/redis/modules/redisgraph.so" >> redis.conf \ + && echo "loadmodule /usr/lib/redis/modules/redisearch.so" >> redis.conf \ + && echo "dir /proofscape/graphdb/re" >> redis.conf \ + && echo "save 1 1" >> redis.conf + +{% else %} + RUN echo "# For now, we need no special config." >> redis.conf RUN echo "# If and when we do, use this file." >> redis.conf +{% endif %} + # Use the One-Container App config: ENV FLASK_CONFIG OCA + +{% if gdb=="re" %} +ENV GRAPHDB_URI redis://localhost:6379 +{% else %} +ENV GRAPHDB_URI file:///home/pfsc/proofscape/graphdb/gl/gremlite.db +{% endif %} + # Set to accept user config from PFSC_ROOT/deploy/pfsc.conf: ENV LOAD_PFSC_CONF_FROM_STANDARD_DEPLOY_DIR 1 # Want to distribute an image that's ready to work with some of the more diff --git a/server/config.py b/server/config.py index 0a0478f..e681411 100644 --- a/server/config.py +++ b/server/config.py @@ -686,7 +686,11 @@ class OcaConfig(DockerDevConfig): RECORD_PER_USER_TRUST_SETTINGS = True REDIS_URI = "redis://localhost:6379" SOCKETIO_MESSAGE_QUEUE = "redis://localhost:6379" - GRAPHDB_URI = "file:///home/pfsc/proofscape/graphdb/gl/gremlite.db" + + # As long as we are distributing two different versions of the OCA, one that uses + # GremLite and one that uses RedisGraph, we read the GRAPHDB_URI from the environment. + # When we go back to distributing only one, we can once again hard-code the value here. + GRAPHDB_URI = os.getenv("GRAPHDB_URI") # The app URL prefix is important, so that the URL at which the ISE loads, # `localhost:7372/ProofscapeISE` is recognizable by the PBE (Proofscape From f1db101cbed9ec6999a8e1d2cbc8e3976d924e98 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Mon, 23 Feb 2026 07:20:35 -0500 Subject: [PATCH 32/35] Relax constraint for `-dev` in image tag Still must be present, but no longer required to be at end. This makes room for the `-gl` and `-rg` suffixes during development. --- manage/tools/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manage/tools/build.py b/manage/tools/build.py index e2ab858..f5c47c0 100644 --- a/manage/tools/build.py +++ b/manage/tools/build.py @@ -155,9 +155,9 @@ def write_dockerignore_for_pyc(): def build_license_info_dict(skip_licensing: bool, tag: str) -> dict: skip_key = 'Deliberately skipping license file generation for development purposes.' if skip_licensing: - if not tag.endswith('-dev'): + if 'dev' not in tag.split('-'): raise click.UsageError( - 'When using the --skip-licensing switch, your TAG must end with "-dev".\n' + 'When using the --skip-licensing switch, your TAG must contain "-dev".\n' 'The purpose of the switch is to let you skip updating license info when\n' 'trying out new packages (or package upgrades) during development.' ) From 78a9edbd8c1efd9d2a76a34785f49da7102d5603 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Mon, 23 Feb 2026 07:20:53 -0500 Subject: [PATCH 33/35] Set dev version at 0.30.4 instead of 0.31.0 --- client/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/package.json b/client/package.json index 4504ccd..ad04b30 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "@proofscape/pise-client", - "version": "0.31.0-dev", + "version": "0.30.4-dev", "license": "Apache-2.0", "description": "Client-side code for PISE, the Proofscape Integrated Study Environment", "repository": { From 9d7cff5491ed2e17ee8a001156ab395f58389641 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Wed, 25 Feb 2026 10:04:05 -0500 Subject: [PATCH 34/35] Workflows: build, test, and publish two versions of OCA (-rg and -gl) --- .github/workflows/pise-build-and-test.yml | 116 +++++++++++++++++++++- .github/workflows/pise-publish.yml | 14 ++- 2 files changed, 125 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pise-build-and-test.yml b/.github/workflows/pise-build-and-test.yml index 32b7186..9f9812c 100644 --- a/.github/workflows/pise-build-and-test.yml +++ b/.github/workflows/pise-build-and-test.yml @@ -236,6 +236,9 @@ jobs: echo "USE_BASE_NGINX_FRONTEND=0" >> $GITHUB_ENV echo "${{inputs.v-tag}}" | sed 's/v/PISE_VERS=/' >> $GITHUB_ENV echo "DEBUG_WORKFLOW=${{ inputs.debug-level }}" >> $GITHUB_ENV + - name: Set OCA-gl tag + run: | + echo "PISE_OCA_GL_TAG=${{ format('{0}-gl', env.PISE_VERS) }}" >> $GITHUB_ENV # ----------------------------------------- # INSTALL # ----------------------------------------- @@ -499,7 +502,7 @@ jobs: pfsc deploy generate --gdb gl \ ${{ !inputs.pub-prep && '--mount-code' || '--no-mount-code' }} \ --pfsc-tag ${{ env.PISE_VERS }} \ - --oca-tag ${{ env.PISE_VERS }} \ + --oca-tag ${{ env.PISE_OCA_GL_TAG }} \ -n 1 --demos \ --dirname ${{ env.TEST_DEPLOY_DIR }} \ --flask-config dockerdev \ @@ -701,7 +704,7 @@ jobs: name: Upload selenium results uses: actions/upload-artifact@v4 with: - name: selenium_results + name: debug_selenium_results path: ${{ env.PFSC_ROOT }}/selenium_results # If not doing pub prep, at least dump logs for manual inspection - if: ${{ always() && !inputs.pub-prep }} @@ -738,13 +741,118 @@ jobs: # ----------------------------------------- # OCA # ----------------------------------------- + # ------------------------------------------------------------------------------------------- + # As long as we are distributing two OCA images (one with GremLite and one with RedisGraph) we + # have this next section to build and test the RedisGraph image. + - if: ${{ inputs.pub-prep }} + name: Set up vars for OCA-rg build + run: | + echo "PISE_OCA_RG_TAG=${{ format('{0}-rg', env.PISE_VERS) }}" >> $GITHUB_ENV + echo "PISE_OCA_RG_CTX_TAR=${{ format('{0}/contexts/pise-oca-rg-context.tar.gz', github.workspace) }}" >> $GITHUB_ENV + echo "PISE_OCA_RG_CTX_DIR=${{ format('{0}/contexts/pise-oca-rg-context', github.workspace) }}" >> $GITHUB_ENV + echo "TEST_RG_DEPLOY_DIR=testing-rg" >> $GITHUB_ENV + - if: ${{ inputs.pub-prep }} + name: Generate RG deployment directory + working-directory: pise/manage + run: | + source venv/bin/activate + pfsc deploy generate --gdb re \ + --no-local --no-mount-code \ + --pfsc-tag ${{ env.PISE_VERS }} \ + --oca-tag ${{ env.PISE_OCA_RG_TAG }} \ + -n 1 --demos \ + --dirname ${{ env.TEST_RG_DEPLOY_DIR }} \ + --flask-config dockerdev \ + --no-per-deploy-dirs \ + --lib-vol=${{ env.LIB_VOLUME }} \ + --build-vol=${{ env.BUILD_VOLUME }} \ + --gdb-vol=${{ env.GDB_VOLUME }} + - if: ${{ inputs.pub-prep }} + name: Make pise OCA-rg build context + working-directory: pise/manage + run: | + source venv/bin/activate + pfsc build oca --gdb=re --skip-licensing --tar-path=${{env.PISE_OCA_RG_CTX_TAR}} ${{ env.PISE_OCA_RG_TAG }} + mkdir ${{env.PISE_OCA_RG_CTX_DIR}} + tar -x -f ${{env.PISE_OCA_RG_CTX_TAR}} -C ${{env.PISE_OCA_RG_CTX_DIR}} + - if: ${{ inputs.pub-prep }} + name: Build and export pise OCA-rg image to Docker + uses: docker/build-push-action@v4 + with: + context: ${{env.PISE_OCA_RG_CTX_DIR}} + load: true + tags: pise:${{env.PISE_OCA_RG_TAG}} + cache-to: type=gha + cache-from: type=gha + # ----------------------------------------- + # We remake the lib, build, gdb volumes fresh for use with the OCA. + # First remove the existing ones + - if: ${{ inputs.pub-prep }} + name: Remove existing lib, build, gdb volumes + run: docker volume rm ${{env.LIB_VOLUME}} ${{env.BUILD_VOLUME}} ${{env.GDB_VOLUME}} + # Form and populate a `lib` volume + - if: ${{ inputs.pub-prep }} + name: Form and populate lib volume for OCA + run: | + docker run --rm --entrypoint=bash \ + -v ${{env.PFSC_ROOT}}/lib:/usr/local/share/proofscape_lib:ro \ + --mount 'type=volume,src=${{env.LIB_VOLUME}},dst=/proofscape/lib' \ + pise-server:${{ env.PISE_VERS }} \ + -c "cp -r /usr/local/share/proofscape_lib/* ~/proofscape/lib" + # Check population of lib volume + - if: ${{ inputs.pub-prep && env.DEBUG_WORKFLOW >= 1 }} + name: Check population of lib volume for OCA + run: | + docker run --rm --entrypoint=bash \ + --mount 'type=volume,src=${{env.LIB_VOLUME}},dst=/proofscape/lib' \ + pise-server:${{ env.PISE_VERS }} \ + -c "cd ~/proofscape/lib; ls -l; ls -l test/hist/lit" + # Form `build` and `gdb` volumes + - if: ${{ inputs.pub-prep }} + name: Form build and gdb volumes for OCA + run: | + docker volume create ${{env.BUILD_VOLUME}} + docker volume create ${{env.GDB_VOLUME}} + # ----------------------------------------- + # Run the OCA-rg + - if: ${{ inputs.pub-prep }} + name: Run the OCA-rg + working-directory: ${{ format('{0}/deploy/{1}', env.PFSC_ROOT, env.TEST_RG_DEPLOY_DIR) }} + run: docker compose -f oca-docker-compose.yml up -d + # ----------------------------------------- + # Run OCA-rg Selenium tests + - if: ${{ inputs.pub-prep }} + name: Run OCA-rg Selenium tests + working-directory: pise/manage + run: | + source venv/bin/activate + pytest tests/selenium/test_basic_run_oca.py + - if: ${{ inputs.pub-prep && always() }} + name: Record docker logs for OCA-rg + working-directory: ${{ env.PFSC_ROOT }} + run: | + mkdir -p selenium_results/docker_logs + docker logs ${{env.TEST_RG_DEPLOY_DIR}}-pise-1 > selenium_results/docker_logs/pise-1-rg.txt 2>&1 + docker exec ${{env.TEST_RG_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/pfsc_web-stdout*.log" > selenium_results/docker_logs/pise-1-web-rg-stdout.txt 2>&1 + docker exec ${{env.TEST_RG_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/pfsc_web-stderr*.log" > selenium_results/docker_logs/pise-1-web-rg-stderr.txt 2>&1 + docker exec ${{env.TEST_RG_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/redisgraph-stdout*.log" > selenium_results/docker_logs/pise-1-redisgraph-stdout.txt 2>&1 + docker exec ${{env.TEST_RG_DEPLOY_DIR}}-pise-1 bash -c "cat /tmp/redisgraph-stderr*.log" > selenium_results/docker_logs/pise-1-redisgraph-stderr.txt 2>&1 + # ----------------------------------------- + # Stop the OCA-rg + - if: ${{ inputs.pub-prep }} + name: Stop the OCA-rg + working-directory: ${{ format('{0}/deploy/{1}', env.PFSC_ROOT, env.TEST_RG_DEPLOY_DIR) }} + run: docker compose -f oca-docker-compose.yml down + # End of auxiliary OCA section + # ------------------------------------------------------------------------------------------- + # ----------------------------------------- # If pub-prep, build pise (OCA) image - if: ${{ inputs.pub-prep }} name: Make pise OCA build context working-directory: pise/manage run: | source venv/bin/activate - pfsc build oca --tar-path=${{env.PISE_OCA_CTX_TAR}} ${{env.PISE_VERS}} + pfsc build oca --gdb=gl --tar-path=${{env.PISE_OCA_CTX_TAR}} ${{env.PISE_OCA_GL_TAG}} mkdir ${{env.PISE_OCA_CTX_DIR}} tar -x -f ${{env.PISE_OCA_CTX_TAR}} -C ${{env.PISE_OCA_CTX_DIR}} - if: ${{ inputs.pub-prep }} @@ -753,7 +861,7 @@ jobs: with: context: ${{env.PISE_OCA_CTX_DIR}} load: true - tags: pise:${{env.PISE_VERS}} + tags: pise:${{env.PISE_OCA_GL_TAG}} cache-to: type=gha cache-from: type=gha # Check contents of image's working directory diff --git a/.github/workflows/pise-publish.yml b/.github/workflows/pise-publish.yml index 4ee9844..e195221 100644 --- a/.github/workflows/pise-publish.yml +++ b/.github/workflows/pise-publish.yml @@ -103,6 +103,7 @@ jobs: tar -x -f pise-server-context.tar.gz -C pise-server-context tar -x -f pise-frontend-context.tar.gz -C pise-frontend-context tar -x -f pise-oca-context.tar.gz -C pise-oca-context + tar -x -f pise-oca-rg-context.tar.gz -C pise-oca-rg-context - name: Login to Docker Hub uses: docker/login-action@v2 with: @@ -138,5 +139,16 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: | - ${{env.DH_NAMESPACE}}/pise:${{env.PISE_VERS}} + ${{env.DH_NAMESPACE}}/pise:${{ format('{0}-gl', env.PISE_VERS) }} + # Note: While we are publishing the RG version of the OCA, that is the one + # that gets the secondary tag ('latest' or 'edge'), for backwards-compatibility. + - if: ${{ steps.switch.outputs.pub-dh == 'yes' }} + name: Publish pise OCA-rg to Docker Hub + uses: docker/build-push-action@v4 + with: + context: contexts/pise-oca-rg-context + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{env.DH_NAMESPACE}}/pise:${{ format('{0}-rg', env.PISE_VERS) }} ${{env.DH_NAMESPACE}}/pise:${{env.SECOND_TAG}} From b43314b0f80b6ef3b3af3fb7bec78ee59e089347 Mon Sep 17 00:00:00 2001 From: Steve Kieffer Date: Fri, 27 Feb 2026 07:24:21 -0500 Subject: [PATCH 35/35] Add comment explaining base image choices for OCA --- manage/topics/pfsc/templates/Dockerfile.oca | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/manage/topics/pfsc/templates/Dockerfile.oca b/manage/topics/pfsc/templates/Dockerfile.oca index 7091d9b..220c720 100644 --- a/manage/topics/pfsc/templates/Dockerfile.oca +++ b/manage/topics/pfsc/templates/Dockerfile.oca @@ -16,6 +16,16 @@ {% if gdb=="re" %} +# Note on the Python base image tag 3.11.13-slim-bullseye used here: +# RedisGraph was last supported on a version of Redis that requires libssl1.1, +# and we therefore cannot easily move from Debian Bullseye up to Bookworm, since that +# comes with an upgrade to libssl3. As of now, a Python 3.11.14 image has not been +# released for Bullseye; the latest 3.11.x is 3.11.13, so we use that. +# Meanwhile, GremLite cannot be operated on this OCA image (which might be nice, +# for some automated transition tools) since it requires libsqlite 3.35 or greater, +# but Bullseye has an older version. We could think about installing custom, secondary +# versions of the libssl and/or libsqlite libraries, but I'm putting that off for another +# time, if ever. For now, we just want to work with ready-made base images. FROM --platform={{platform}} redis/redis-stack-server:{{redisgraph_image_tag}} AS rg FROM --platform={{platform}} python:3.11.13-slim-bullseye AS basis ARG DEBIAN_FRONTEND=noninteractive