From 8cca8060d6006a5c39468b636a07e68bf81d9ebb Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 14:05:57 -0600 Subject: [PATCH 01/78] Started building out RemoteFileSource plugin (#DH-20578) --- plugin/remotefilesource/build.gradle | 14 ++++ plugin/remotefilesource/gradle.properties | 1 + .../RemoteFileSourceCommandResolver.java | 79 +++++++++++++++++++ .../RemoteFileSourceServicePlugin.java | 61 ++++++++++++++ proto/proto-backplane-grpc/Dockerfile | 16 ++++ .../proto/remotefilesource.proto | 36 +++++++++ settings.gradle | 3 + 7 files changed, 210 insertions(+) create mode 100644 plugin/remotefilesource/build.gradle create mode 100644 plugin/remotefilesource/gradle.properties create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java create mode 100644 proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto diff --git a/plugin/remotefilesource/build.gradle b/plugin/remotefilesource/build.gradle new file mode 100644 index 00000000000..61e914c8f88 --- /dev/null +++ b/plugin/remotefilesource/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'io.deephaven.project.register' +} + +dependencies { + implementation project(':plugin') + implementation project(':server') + implementation project(':proto:proto-backplane-grpc') + implementation project(':proto:proto-backplane-grpc-flight') + + compileOnly libs.autoservice + compileOnly libs.jetbrains.annotations + annotationProcessor libs.autoservice.compiler +} \ No newline at end of file diff --git a/plugin/remotefilesource/gradle.properties b/plugin/remotefilesource/gradle.properties new file mode 100644 index 00000000000..c186bbfdde1 --- /dev/null +++ b/plugin/remotefilesource/gradle.properties @@ -0,0 +1 @@ +io.deephaven.project.ProjectType=JAVA_PUBLIC diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java new file mode 100644 index 00000000000..669a6130798 --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -0,0 +1,79 @@ +package io.deephaven.remotefilesource; + +import io.deephaven.server.session.CommandResolver; +import io.deephaven.server.session.SessionState; +import io.deephaven.server.session.TicketRouter; +import io.deephaven.server.session.WantsTicketRouter; + +import org.apache.arrow.flight.impl.Flight; +import org.jetbrains.annotations.Nullable; + +import java.nio.ByteBuffer; +import java.util.function.Consumer; + +public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTicketRouter { + @Override + public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, + final Flight.FlightDescriptor descriptor, + final String logId) { + return null; + } + + @Override + public void forAllFlightInfo(@Nullable final SessionState session, final Consumer visitor) { + // nothing to do + } + + @Override + public String getLogNameFor(final ByteBuffer ticket, final String logId) { + // no tickets + throw new UnsupportedOperationException(); + } + + @Override + public boolean handlesCommand(final Flight.FlightDescriptor descriptor) { + return false; + } + + @Override + public SessionState.ExportBuilder publish(final SessionState session, + final ByteBuffer ticket, + final String logId, + @Nullable final Runnable onPublish) { + // no publishing + throw new UnsupportedOperationException(); + } + + @Override + public SessionState.ExportBuilder publish(final SessionState session, + final Flight.FlightDescriptor descriptor, final String logId, + @Nullable final Runnable onPublish) { + // no publishing + throw new UnsupportedOperationException(); + } + + @Override + public SessionState.ExportObject resolve(@Nullable final SessionState session, + final Flight.FlightDescriptor descriptor, + final String logId) { + return null; + } + + @Override + public SessionState.ExportObject resolve(@Nullable final SessionState session, + final ByteBuffer ticket, + final String logId) { + // no tickets + throw new UnsupportedOperationException(); + } + + @Override + public void setTicketRouter(TicketRouter ticketRouter) { + // not needed + } + + @Override + public byte ticketRoute() { + return 0; + } +} diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java new file mode 100644 index 00000000000..9fcf6bc8505 --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -0,0 +1,61 @@ +package io.deephaven.remotefilesource; + +import com.google.auto.service.AutoService; +import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.plugin.type.ObjectType; +import io.deephaven.plugin.type.ObjectTypeBase; +import io.deephaven.plugin.type.ObjectCommunicationException; +import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; + +import java.nio.ByteBuffer; + +@AutoService(ObjectType.class) +public class RemoteFileSourceServicePlugin extends ObjectTypeBase { + public RemoteFileSourceServicePlugin() {} + + @Override + public String name() { + return "RemoteFileSourceService"; + } + + @Override + public boolean isType(Object object) { + return object instanceof RemoteFileSourceServicePlugin; + } + + @Override + public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { + connection.onData(ByteBuffer.allocate(0)); + return new RemoteFileSourceMessageStream(connection); + } + + /** + * A message stream for the RemoteFileSourceService. + */ + private class RemoteFileSourceMessageStream implements MessageStream { + private final MessageStream connection; + + public RemoteFileSourceMessageStream(final MessageStream connection) { + this.connection = connection; + } + + @Override + public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { + final RemoteFileSourcePluginFetchRequest request; + + try { + request = RemoteFileSourcePluginFetchRequest.parseFrom(payload); + } catch (InvalidProtocolBufferException e) { + // There is no identifier here, so we cannot properly return an error that is bound to the request. + // Instead, we throw an Exception causing the server to close the entire MessageStream and + // propagate a general error to the client. + throw new RuntimeException(e); + } + } + + @Override + public void onClose() { + + } + } +} diff --git a/proto/proto-backplane-grpc/Dockerfile b/proto/proto-backplane-grpc/Dockerfile index a95d0465655..0b142c717a0 100644 --- a/proto/proto-backplane-grpc/Dockerfile +++ b/proto/proto-backplane-grpc/Dockerfile @@ -29,6 +29,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -48,6 +49,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -64,6 +66,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -83,6 +86,7 @@ RUN set -eux; \ /includes/deephaven_core/proto/application.proto \ /includes/deephaven_core/proto/inputtable.proto \ /includes/deephaven_core/proto/partitionedtable.proto \ + /includes/deephaven_core/proto/remotefilesource.proto \ /includes/deephaven_core/proto/config.proto \ /includes/deephaven_core/proto/hierarchicaltable.proto \ /includes/deephaven_core/proto/storage.proto; \ @@ -151,6 +155,12 @@ RUN set -eux; \ --doc_opt=html,partitionedtable.html \ -I/includes \ /includes/deephaven_core/proto/partitionedtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-html \ + --doc_opt=html,remotefilesource.html \ + -I/includes \ + /includes/deephaven_core/proto/remotefilesource.proto; \ /opt/protoc/bin/protoc \ --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ --doc_out=generated/proto-doc/multi-html \ @@ -218,6 +228,12 @@ RUN set -eux; \ --doc_opt=markdown,partitionedtable.md \ -I/includes \ /includes/deephaven_core/proto/partitionedtable.proto; \ + /opt/protoc/bin/protoc \ + --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ + --doc_out=generated/proto-doc/multi-md \ + --doc_opt=markdown,remotefilesource.md \ + -I/includes \ + /includes/deephaven_core/proto/remotefilesource.proto; \ /opt/protoc/bin/protoc \ --plugin=protoc-gen-doc=/usr/local/bin/protoc-gen-doc \ --doc_out=generated/proto-doc/multi-md \ diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto new file mode 100644 index 00000000000..586630c3c1f --- /dev/null +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + */ +syntax = "proto3"; + +package io.deephaven.proto.backplane.grpc; + +option java_multiple_files = true; +option optimize_for = SPEED; +option go_package = "github.com/deephaven/deephaven-core/go/internal/proto/remotefilesource"; + +import "deephaven_core/proto/ticket.proto"; + +message RemoteFileSourcePluginRequest { + // A client-specified identifier used to identify the response for this request when multiple requests are in flight. + // This must be set to a unique value. + string request_id = 1; + oneof request { + RemoteFileSourceMetaRequest meta = 2; + RemoteFileSourceSetConnectionIdRequest set_connection_id = 3; + } +} + +// Request meta info from the server plugin +message RemoteFileSourceMetaRequest { +} + +message RemoteFileSourceSetConnectionIdRequest { + // The connection ID to set for the remote file source. + string connection_id = 1; +} + +//// Fetch the remote file source plugin into the specified ticket +//message RemoteFileSourcePluginFetchRequest { +// io.deephaven.proto.backplane.grpc.Ticket result_id = 1; +//} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 8b7e4e3c4d7..77ca2722a3f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -318,6 +318,9 @@ project(':plugin-figure').projectDir = file('plugin/figure') include(':plugin-partitionedtable') project(':plugin-partitionedtable').projectDir = file('plugin/partitionedtable') +include(':plugin-remotefilesource') +project(':plugin-remotefilesource').projectDir = file('plugin/remotefilesource') + include(':plugin-hierarchicaltable') project(':plugin-hierarchicaltable').projectDir = file('plugin/hierarchicaltable') From b0871b772ddd5981fc2df336b132b416ce486ed4 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 14:30:52 -0600 Subject: [PATCH 02/78] Command resolver now fetches plugin (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 669a6130798..d56dd9ca00c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -1,10 +1,23 @@ package io.deephaven.remotefilesource; +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.Code; +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.verify.Assert; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; +import io.deephaven.proto.backplane.grpc.Ticket; +import io.deephaven.proto.util.Exceptions; import io.deephaven.server.session.CommandResolver; import io.deephaven.server.session.SessionState; import io.deephaven.server.session.TicketRouter; import io.deephaven.server.session.WantsTicketRouter; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; import org.apache.arrow.flight.impl.Flight; import org.jetbrains.annotations.Nullable; @@ -12,11 +25,83 @@ import java.util.function.Consumer; public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTicketRouter { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceCommandResolver.class); + + private static final String FETCH_PLUGIN_TYPE_URL = + "type.googleapis.com/" + RemoteFileSourcePluginFetchRequest.getDescriptor().getFullName(); + + private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any command) { + if (!FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl())) { + throw new IllegalArgumentException("Not a valid remotefilesource command: " + command.getTypeUrl()); + } + + final ByteString bytes = command.getValue(); + final RemoteFileSourcePluginFetchRequest request; + try { + request = RemoteFileSourcePluginFetchRequest.parseFrom(bytes); + } catch (InvalidProtocolBufferException e) { + throw new UncheckedDeephavenException("Could not parse RemoteFileSourcePluginFetchRequest", e); + } + return request; + } + + private static Any parseOrNull(final ByteString data) { + try { + return Any.parseFrom(data); + } catch (final InvalidProtocolBufferException e) { + return null; + } + } + + public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { + final Ticket resultTicket = request.getResultId(); + final boolean hasResultId = !resultTicket.getTicket().isEmpty(); + if (!hasResultId) { + throw new StatusRuntimeException(Status.INVALID_ARGUMENT); + } + + final SessionState.ExportBuilder pluginExportBuilder = + session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); + pluginExportBuilder.require(); + + final SessionState.ExportObject pluginExport = + pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); + + final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() + .setFlightDescriptor(descriptor) + .addEndpoint(Flight.FlightEndpoint.newBuilder() + .setTicket(Flight.Ticket.newBuilder() + .setTicket( + resultTicket.getTicket())) + .build()) + .setTotalRecords(-1) + .setTotalBytes(-1) + .build(); + return SessionState.wrapAsExport(flightInfo); + } + @Override public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final String logId) { - return null; + if (session == null) { + throw new StatusRuntimeException(Status.UNAUTHENTICATED); + } + + final Any request = parseOrNull(descriptor.getCmd()); + if (request == null) { + log.error().append("Could not parse remotefilesource command.").endl(); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Could not parse remotefilesource command Any."); + } + + if (FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { + return fetchPlugin(session, descriptor, parseFetchRequest(request)); + } + + log.error().append("Invalid pivot command typeUrl: " + request.getTypeUrl()).endl(); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Invalid typeUrl: " + request.getTypeUrl()); } @Override @@ -32,7 +117,17 @@ public String getLogNameFor(final ByteBuffer ticket, final String logId) { @Override public boolean handlesCommand(final Flight.FlightDescriptor descriptor) { - return false; + // If not CMD, there is an error with io.deephaven.server.session.TicketRouter.getPathResolver / handlesPath + Assert.eq(descriptor.getType(), "descriptor.getType()", Flight.FlightDescriptor.DescriptorType.CMD, "CMD"); + + // No good way to check if this is a valid command without parsing to Any first. + final Any command = parseOrNull(descriptor.getCmd()); + if (command == null) { + return false; + } + + // Check if the command matches any types that this resolver handles. + return FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl()); } @Override From e183397500f1fb2e5e40ee1f19d29cd5f1f9ae63 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 14:34:38 -0600 Subject: [PATCH 03/78] RemoteFileSourceTicketResolverFactoryService (#DH-20578) --- ...emoteFileSourceTicketResolverFactoryService.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java new file mode 100644 index 00000000000..89fa503b9aa --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -0,0 +1,13 @@ +package io.deephaven.remotefilesource; + +import com.google.auto.service.AutoService; +import io.deephaven.server.runner.TicketResolversFromServiceLoader; +import io.deephaven.server.session.TicketResolver; + +@AutoService(TicketResolversFromServiceLoader.Factory.class) +public class RemoteFileSourceTicketResolverFactoryService implements TicketResolversFromServiceLoader.Factory { + @Override + public TicketResolver create(final TicketResolversFromServiceLoader.TicketResolverOptions options) { + return new RemoteFileSourceCommandResolver(); + } +} From 369e6363646ff5c5cd584a721db60175a83c7325 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 26 Nov 2025 16:39:42 -0600 Subject: [PATCH 04/78] Added stub for JsRemoteFileSourceService (#DH-20578) --- .../deephaven_core/proto/remotefilesource.proto | 8 ++++---- .../remotefilesource/JsRemoteFileSourceService.java | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 586630c3c1f..65c28479286 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -30,7 +30,7 @@ message RemoteFileSourceSetConnectionIdRequest { string connection_id = 1; } -//// Fetch the remote file source plugin into the specified ticket -//message RemoteFileSourcePluginFetchRequest { -// io.deephaven.proto.backplane.grpc.Ticket result_id = 1; -//} \ No newline at end of file +// Fetch the remote file source plugin into the specified ticket +message RemoteFileSourcePluginFetchRequest { + io.deephaven.proto.backplane.grpc.Ticket result_id = 1; +} \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java new file mode 100644 index 00000000000..d7df99933b0 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -0,0 +1,12 @@ +package io.deephaven.web.client.api.remotefilesource; + +import io.deephaven.web.client.api.event.HasEventHandling; +import jsinterop.annotations.JsType; + +/** + * JavaScript client for the RemoteFileSource service. + */ +@JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") +public class JsRemoteFileSourceService extends HasEventHandling { + // TODO: This needs to send a RemoteFileSourcePluginFetchRequest flight message to get a plugin service instance +} From a093cdea527b992c485ac9f2cd6ac1bd253e7998 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Wed, 26 Nov 2025 20:16:07 -0600 Subject: [PATCH 05/78] Generated GWT bindings --- .../RemoteFileSourceMetaRequest.java | 30 +++ .../RemoteFileSourcePluginFetchRequest.java | 193 ++++++++++++++++++ .../RemoteFileSourcePluginRequest.java | 143 +++++++++++++ ...emoteFileSourceSetConnectionIdRequest.java | 68 ++++++ .../RequestCase.java | 17 ++ 5 files changed, 451 insertions(+) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java new file mode 100644 index 00000000000..c2d0cc39470 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -0,0 +1,30 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceMetaRequest { + public static native RemoteFileSourceMetaRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( + RemoteFileSourceMetaRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceMetaRequest message, Object writer); + + public static native Object toObject(boolean includeInstance, RemoteFileSourceMetaRequest msg); + + public native Uint8Array serializeBinary(); + + public native Object toObject(); + + public native Object toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java new file mode 100644 index 00000000000..981a4d40580 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -0,0 +1,193 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourcePluginFetchRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ResultIdFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetTicketUnionType { + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType.GetTicketUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType.GetTicketUnionType getTicket(); + + @JsProperty + void setTicket( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType.GetTicketUnionType ticket); + + @JsOverlay + default void setTicket(String ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + + @JsOverlay + default void setTicket(Uint8Array ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); + + @JsProperty + void setResultId( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ResultIdFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetTicketUnionType { + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType.GetTicketUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType.GetTicketUnionType getTicket(); + + @JsProperty + void setTicket( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType.GetTicketUnionType ticket); + + @JsOverlay + default void setTicket(String ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + + @JsOverlay + default void setTicket(Uint8Array ticket) { + setTicket( + Js.uncheckedCast( + ticket)); + } + } + + @JsOverlay + static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); + + @JsProperty + void setResultId( + RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); + } + + public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourcePluginFetchRequest deserializeBinaryFromReader( + RemoteFileSourcePluginFetchRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourcePluginFetchRequest message, Object writer); + + public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourcePluginFetchRequest msg); + + public native void clearResultId(); + + public native Ticket getResultId(); + + public native boolean hasResultId(); + + public native Uint8Array serializeBinary(); + + public native void setResultId(); + + public native void setResultId(Ticket value); + + public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject( + boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java new file mode 100644 index 00000000000..1acf26d56de --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java @@ -0,0 +1,143 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourcePluginRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetConnectionIdFieldType { + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + Object getMeta(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType getSetConnectionId(); + + @JsProperty + void setMeta(Object meta); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetConnectionId( + RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType setConnectionId); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetConnectionIdFieldType { + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + @JsOverlay + static RemoteFileSourcePluginRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + Object getMeta(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType getSetConnectionId(); + + @JsProperty + void setMeta(Object meta); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetConnectionId( + RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType setConnectionId); + } + + public static native RemoteFileSourcePluginRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourcePluginRequest deserializeBinaryFromReader( + RemoteFileSourcePluginRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourcePluginRequest message, Object writer); + + public static native RemoteFileSourcePluginRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourcePluginRequest msg); + + public native void clearMeta(); + + public native void clearSetConnectionId(); + + public native RemoteFileSourceMetaRequest getMeta(); + + public native int getRequestCase(); + + public native String getRequestId(); + + public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); + + public native boolean hasMeta(); + + public native boolean hasSetConnectionId(); + + public native Uint8Array serializeBinary(); + + public native void setMeta(); + + public native void setMeta(RemoteFileSourceMetaRequest value); + + public native void setRequestId(String value); + + public native void setSetConnectionId(); + + public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); + + public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java new file mode 100644 index 00000000000..590ba5ee643 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java @@ -0,0 +1,68 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceSetConnectionIdRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getConnectionId(); + + @JsProperty + void setConnectionId(String connectionId); + } + + public static native RemoteFileSourceSetConnectionIdRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceSetConnectionIdRequest deserializeBinaryFromReader( + RemoteFileSourceSetConnectionIdRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceSetConnectionIdRequest message, Object writer); + + public static native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceSetConnectionIdRequest msg); + + public native String getConnectionId(); + + public native Uint8Array serializeBinary(); + + public native void setConnectionId(String value); + + public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject( + boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java new file mode 100644 index 00000000000..c263a6194cb --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java @@ -0,0 +1,17 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourcepluginrequest; + +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest.RequestCase", + namespace = JsPackage.GLOBAL) +public class RequestCase { + public static int META, + REQUEST_NOT_SET, + SET_CONNECTION_ID; +} From 5027810999a679baf23160c201588e5dc8cdbd7f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 3 Dec 2025 17:58:10 -0600 Subject: [PATCH 06/78] Fetching plugin service working (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 1 + proto/raw-js-openapi/src/index.js | 2 + .../src/shim/remotefilesource_pb.js | 2 + proto/raw-js-openapi/webpack.config.js | 2 +- server/jetty-app-11/build.gradle | 1 + server/jetty-app/build.gradle | 1 + .../deephaven/web/client/api/CoreClient.java | 5 + .../JsRemoteFileSourceService.java | 143 +++++++++++++++++- 8 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 proto/raw-js-openapi/src/shim/remotefilesource_pb.js diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index d56dd9ca00c..ac2e7b0ba72 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -30,6 +30,7 @@ public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTi private static final String FETCH_PLUGIN_TYPE_URL = "type.googleapis.com/" + RemoteFileSourcePluginFetchRequest.getDescriptor().getFullName(); + private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any command) { if (!FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl())) { throw new IllegalArgumentException("Not a valid remotefilesource command: " + command.getTypeUrl()); diff --git a/proto/raw-js-openapi/src/index.js b/proto/raw-js-openapi/src/index.js index 6d7b2e69ff5..9176663e2b6 100644 --- a/proto/raw-js-openapi/src/index.js +++ b/proto/raw-js-openapi/src/index.js @@ -6,6 +6,7 @@ var application_pb = require("deephaven_core/proto/application_pb"); var inputtable_pb = require("deephaven_core/proto/inputtable_pb"); var object_pb = require("deephaven_core/proto/object_pb"); var partitionedtable_pb = require("deephaven_core/proto/partitionedtable_pb"); +var remotefilesource_pb = require("deephaven_core/proto/remotefilesource_pb"); var storage_pb = require("deephaven_core/proto/storage_pb"); var config_pb = require("deephaven_core/proto/config_pb"); var hierarchicaltable_pb = require("deephaven_core/proto/hierarchicaltable_pb"); @@ -46,6 +47,7 @@ var io = { deephaven_core: { object_pb_service, partitionedtable_pb, partitionedtable_pb_service, + remotefilesource_pb, storage_pb, storage_pb_service, config_pb, diff --git a/proto/raw-js-openapi/src/shim/remotefilesource_pb.js b/proto/raw-js-openapi/src/shim/remotefilesource_pb.js new file mode 100644 index 00000000000..01f90ee66e7 --- /dev/null +++ b/proto/raw-js-openapi/src/shim/remotefilesource_pb.js @@ -0,0 +1,2 @@ +Object.assign(exports, require('real/remotefilesource_pb').io.deephaven.proto.backplane.grpc) + diff --git a/proto/raw-js-openapi/webpack.config.js b/proto/raw-js-openapi/webpack.config.js index f0e47f9cbee..8d64a5ac4bd 100644 --- a/proto/raw-js-openapi/webpack.config.js +++ b/proto/raw-js-openapi/webpack.config.js @@ -3,7 +3,7 @@ const path = require('path'); // Workaround for broken codegen from protoc-gen-js using import_style=commonjs_strict, both in // the grpc-web protoc-gen-ts plugin, and in protoc-gen-js itself: const aliases = {}; -for (const proto of ['application', 'config', 'console', 'hierarchicaltable', 'inputtable', 'object', 'partitionedtable', 'session', 'storage', 'table', 'ticket']) { +for (const proto of ['application', 'config', 'console', 'hierarchicaltable', 'inputtable', 'object', 'partitionedtable', 'remotefilesource', 'session', 'storage', 'table', 'ticket']) { // Allows a reference to the real proto files, to be made from the shim aliases[`real/${proto}_pb`] = `${__dirname}/build/js-src/deephaven_core/proto/${proto}_pb`; diff --git a/server/jetty-app-11/build.gradle b/server/jetty-app-11/build.gradle index e075c08a8fa..3a4680c88ca 100644 --- a/server/jetty-app-11/build.gradle +++ b/server/jetty-app-11/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':server-jetty-11') implementation project(':extensions-flight-sql') + implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler diff --git a/server/jetty-app/build.gradle b/server/jetty-app/build.gradle index 50c7406be2b..728cddc629a 100644 --- a/server/jetty-app/build.gradle +++ b/server/jetty-app/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':server-jetty') implementation project(':extensions-flight-sql') + implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 0abbdd30efe..7a175c7b307 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -15,6 +15,7 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.config_pb_service.ConfigServiceClient; import io.deephaven.javascript.proto.dhinternal.jspb.Map; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.client.api.remotefilesource.JsRemoteFileSourceService; import io.deephaven.web.client.api.storage.JsStorageService; import io.deephaven.web.client.fu.JsLog; import io.deephaven.web.client.fu.LazyPromise; @@ -154,6 +155,10 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } + public JsRemoteFileSourceService getRemoteFileSourceService() { + return new JsRemoteFileSourceService(ideConnection.connection.get()); + } + public Promise getAsIdeConnection() { return Promise.resolve(ideConnection); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index d7df99933b0..e2c0f843083 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,6 +1,19 @@ package io.deephaven.web.client.api.remotefilesource; +import elemental2.core.Uint8Array; +import elemental2.promise.Promise; +import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; +import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; +import io.deephaven.web.client.api.Callbacks; +import io.deephaven.web.client.api.ServerObject; +import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.client.api.widget.JsWidgetExportedObject; +import jsinterop.annotations.JsIgnore; +import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsType; /** @@ -8,5 +21,133 @@ */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { - // TODO: This needs to send a RemoteFileSourcePluginFetchRequest flight message to get a plugin service instance + private final WorkerConnection connection; + + @JsIgnore + public JsRemoteFileSourceService(WorkerConnection connection) { + this.connection = connection; + } + + /** + * Fetches a RemoteFileSource plugin instance from the server. + * + * @return a promise that resolves to a ServerObject representing the RemoteFileSource plugin instance + */ + @JsMethod + public Promise fetchPlugin() { + // Create a new export ticket for the result + Ticket resultTicket = connection.getTickets().newExportTicket(); + + // Create the fetch request + RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); + fetchRequest.setResultId(resultTicket); + + // Serialize the request to bytes + Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); + + // Wrap in google.protobuf.Any with the proper typeUrl + Uint8Array anyWrappedBytes = wrapInAny( + "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", + innerRequestBytes + ); + + // Create a FlightDescriptor with the command + FlightDescriptor descriptor = new FlightDescriptor(); + descriptor.setType(FlightDescriptor.DescriptorType.getCMD()); + descriptor.setCmd(anyWrappedBytes); + + // Send the getFlightInfo request + return Callbacks.grpcUnaryPromise(c -> + connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply) + ).then(flightInfo -> { + // The first endpoint should contain the ticket for the plugin instance + if (flightInfo.getEndpointList().length > 0) { + // Get the Arrow Flight ticket from the endpoint + io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = + flightInfo.getEndpointList().getAt(0).getTicket(); + + // Convert the Arrow Flight ticket to a Deephaven ticket + Ticket dhTicket = new Ticket(); + dhTicket.setTicket(flightTicket.getTicket_asU8()); + + // Create a TypedTicket for the plugin instance + TypedTicket typedTicket = new TypedTicket(); + typedTicket.setTicket(dhTicket); + typedTicket.setType("RemoteFileSourceService"); + + // Return a ServerObject wrapper + return Promise.resolve(new JsWidgetExportedObject(connection, typedTicket)); + } else { + return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); + } + }); + } + + /** + * Wraps a protobuf message in a google.protobuf.Any message. + * + * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") + * @param messageBytes the serialized protobuf message bytes + * @return the serialized Any message containing the wrapped message + */ + private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { + // Encode the type_url string to UTF-8 bytes + Uint8Array typeUrlBytes = stringToUtf8(typeUrl); + + // Calculate sizes for protobuf encoding + // Field 1 (type_url): tag + length + data + int typeUrlTag = (1 << 3) | 2; // field 1, wire type 2 (length-delimited) + int typeUrlFieldSize = sizeOfVarint(typeUrlTag) + sizeOfVarint(typeUrlBytes.length) + typeUrlBytes.length; + + // Field 2 (value): tag + length + data + int valueTag = (2 << 3) | 2; // field 2, wire type 2 (length-delimited) + int valueFieldSize = sizeOfVarint(valueTag) + sizeOfVarint(messageBytes.length) + messageBytes.length; + + int totalSize = typeUrlFieldSize + valueFieldSize; + Uint8Array result = new Uint8Array(totalSize); + int pos = 0; + + // Write type_url field + pos = writeVarint(result, pos, typeUrlTag); + pos = writeVarint(result, pos, typeUrlBytes.length); + for (int i = 0; i < typeUrlBytes.length; i++) { + result.setAt(pos++, typeUrlBytes.getAt(i)); + } + + // Write value field + pos = writeVarint(result, pos, valueTag); + pos = writeVarint(result, pos, messageBytes.length); + for (int i = 0; i < messageBytes.length; i++) { + result.setAt(pos++, messageBytes.getAt(i)); + } + + return result; + } + + private static Uint8Array stringToUtf8(String str) { + // Simple UTF-8 encoding for ASCII-compatible strings + Uint8Array bytes = new Uint8Array(str.length()); + for (int i = 0; i < str.length(); i++) { + bytes.setAt(i, (double) str.charAt(i)); + } + return bytes; + } + + private static int sizeOfVarint(int value) { + if (value < 0) return 10; + if (value < 128) return 1; + if (value < 16384) return 2; + if (value < 2097152) return 3; + if (value < 268435456) return 4; + return 5; + } + + private static int writeVarint(Uint8Array buffer, int pos, int value) { + while (value >= 128) { + buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + buffer.setAt(pos++, (double) value); + return pos; + } } From f4f6c3148525ab2ad73b597d6b4002c785da1539 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 09:33:35 -0600 Subject: [PATCH 07/78] Wiring up messagestream (#DH-20578) --- .../deephaven/web/client/api/CoreClient.java | 4 +- .../JsRemoteFileSourceService.java | 148 ++++++++++++++++-- 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 7a175c7b307..833610bd030 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -155,8 +155,8 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public JsRemoteFileSourceService getRemoteFileSourceService() { - return new JsRemoteFileSourceService(ideConnection.connection.get()); + public Promise getRemoteFileSourceService() { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); } public Promise getAsIdeConnection() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index e2c0f843083..5de42dfd637 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,40 +1,72 @@ package io.deephaven.web.client.api.remotefilesource; +import com.vertispan.tsdefs.annotations.TsInterface; +import com.vertispan.tsdefs.annotations.TsName; import elemental2.core.Uint8Array; +import elemental2.dom.DomGlobal; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ConnectRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; -import io.deephaven.web.client.api.ServerObject; import io.deephaven.web.client.api.WorkerConnection; +import io.deephaven.web.client.api.barrage.stream.BiDiStream; import io.deephaven.web.client.api.event.HasEventHandling; -import io.deephaven.web.client.api.widget.JsWidgetExportedObject; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; -import jsinterop.annotations.JsType; +import jsinterop.annotations.JsProperty; +import jsinterop.base.Js; + +import java.util.function.Supplier; /** - * JavaScript client for the RemoteFileSource service. + * JavaScript client for the RemoteFileSource service. Provides bidirectional communication with the server-side + * RemoteFileSourceServicePlugin via a message stream. */ -@JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") +@TsInterface +@TsName(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") + public static final String EVENT_MESSAGE = "message"; + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") + public static final String EVENT_CLOSE = "close"; + private final WorkerConnection connection; + private final TypedTicket typedTicket; + + private final Supplier> streamFactory; + private BiDiStream messageStream; + + private boolean hasFetched; @JsIgnore - public JsRemoteFileSourceService(WorkerConnection connection) { + private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { this.connection = connection; + this.typedTicket = typedTicket; + this.hasFetched = false; + + // Set up the message stream factory + BiDiStream.Factory factory = connection.streamFactory(); + this.streamFactory = () -> factory.create( + connection.objectServiceClient()::messageStream, + (first, headers) -> connection.objectServiceClient().openMessageStream(first, headers), + (next, headers, c) -> connection.objectServiceClient().nextMessageStream(next, headers, c::apply), + new StreamRequest()); } /** - * Fetches a RemoteFileSource plugin instance from the server. + * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. * - * @return a promise that resolves to a ServerObject representing the RemoteFileSource plugin instance + * @param connection the worker connection to use for communication + * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public Promise fetchPlugin() { + public static Promise fetchPlugin(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -75,14 +107,107 @@ public Promise fetchPlugin() { typedTicket.setTicket(dhTicket); typedTicket.setType("RemoteFileSourceService"); - // Return a ServerObject wrapper - return Promise.resolve(new JsWidgetExportedObject(connection, typedTicket)); + // Create a new service instance with the typed ticket and connect to it + JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); + return service.connect(); } else { return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); } }); } + /** + * Establishes the message stream connection to the server-side plugin instance. + * + * @return a promise that resolves to this service instance when the connection is established + */ + @JsIgnore + private Promise connect() { + if (messageStream != null) { + messageStream.end(); + } + + return new Promise<>((resolve, reject) -> { + messageStream = streamFactory.get(); + + messageStream.onData(res -> { + if (!hasFetched) { + hasFetched = true; + resolve.onInvoke(this); + } else { + // Fire message event for subsequent messages + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_MESSAGE, res.getData()); + }, 0); + } + }); + + messageStream.onStatus(status -> { + if (!status.isOk()) { + reject.onInvoke(status.getDetails()); + } + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_CLOSE); + }, 0); + closeStream(); + }); + + messageStream.onEnd(status -> { + closeStream(); + }); + + // First message establishes a connection w/ the plugin object instance we're talking to + StreamRequest req = new StreamRequest(); + ConnectRequest data = new ConnectRequest(); + data.setSourceId(typedTicket); + req.setConnect(data); + messageStream.send(req); + }); + } + + /** + * Sends a message to the server-side plugin. + * + * @param payload the message data to send (string, ArrayBuffer, or typed array) + */ + @JsMethod + public void sendMessage(Object payload) { + if (messageStream == null) { + throw new IllegalStateException("Message stream not connected"); + } + + StreamRequest req = new StreamRequest(); + + // Convert the payload to Uint8Array + Uint8Array data; + if (payload instanceof String) { + data = stringToUtf8((String) payload); + } else if (payload instanceof Uint8Array) { + data = (Uint8Array) payload; + } else { + data = Js.uncheckedCast(payload); + } + + req.getData().setPayload(data); + messageStream.send(req); + } + + /** + * Closes the message stream connection to the server. + */ + @JsMethod + public void close() { + closeStream(); + } + + @JsIgnore + private void closeStream() { + if (messageStream != null) { + messageStream.end(); + messageStream = null; + } + } + /** * Wraps a protobuf message in a google.protobuf.Any message. * @@ -151,3 +276,4 @@ private static int writeVarint(Uint8Array buffer, int pos, int value) { return pos; } } + From 0520366effcadd8ea78ae1ad2021e4f433935ad2 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 11:15:00 -0600 Subject: [PATCH 08/78] test bidirectional communication (#DH-20578) --- .../RemoteFileSourceServicePlugin.java | 161 ++++++++++++++++-- .../proto/remotefilesource.proto | 44 ++++- .../JsRemoteFileSourceService.java | 138 ++++++++++++++- .../RemoteFileSourceClientRequest.java | 59 +++++++ .../RemoteFileSourceMetaRequest.java | 8 +- .../RemoteFileSourceMetaResponse.java | 43 +++++ .../RemoteFileSourcePluginRequest.java | 143 ---------------- .../RemoteFileSourceServerRequest.java | 41 +++++ 8 files changed, 469 insertions(+), 168 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java delete mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 9fcf6bc8505..df99feeca4f 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -2,15 +2,30 @@ import com.google.auto.service.AutoService; import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectType; import io.deephaven.plugin.type.ObjectTypeBase; import io.deephaven.plugin.type.ObjectCommunicationException; -import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; @AutoService(ObjectType.class) public class RemoteFileSourceServicePlugin extends ObjectTypeBase { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); + + private volatile RemoteFileSourceMessageStream messageStream; + public RemoteFileSourceServicePlugin() {} @Override @@ -26,14 +41,36 @@ public boolean isType(Object object) { @Override public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); - return new RemoteFileSourceMessageStream(connection); + messageStream = new RemoteFileSourceMessageStream(connection); + return messageStream; + } + + /** + * Test method to trigger a resource request from the server to the client. + * Can be called from the console to test bidirectional communication. + * + * Usage from console: + * 
+     * service = remote_file_source_service  # The plugin instance
+     * service.testRequestResource("com/example/MyClass.java")
+     *
+ * + * @param resourceName the resource to request from the client + */ + public void testRequestResource(String resourceName) { + if (messageStream == null) { + log.error().append("MessageStream not connected. Please connect a client first.").endl(); + return; + } + messageStream.testRequestResource(resourceName); } /** * A message stream for the RemoteFileSourceService. */ - private class RemoteFileSourceMessageStream implements MessageStream { + private static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; + private final Map> pendingRequests = new ConcurrentHashMap<>(); public RemoteFileSourceMessageStream(final MessageStream connection) { this.connection = connection; @@ -41,21 +78,125 @@ public RemoteFileSourceMessageStream(final MessageStream connection) { @Override public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { - final RemoteFileSourcePluginFetchRequest request; - try { - request = RemoteFileSourcePluginFetchRequest.parseFrom(payload); + // Parse as RemoteFileSourceClientRequest proto (client→server) + byte[] bytes = new byte[payload.remaining()]; + payload.get(bytes); + RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); + + String requestId = message.getRequestId(); + + if (message.hasMetaResponse()) { + // Client is responding to a resource request + RemoteFileSourceMetaResponse response = message.getMetaResponse(); + + CompletableFuture future = pendingRequests.remove(requestId); + if (future != null) { + byte[] content = response.getContent().toByteArray(); + + log.info().append("Received resource response for requestId: ").append(requestId) + .append(", found: ").append(response.getFound()) + .append(", content length: ").append(content.length).endl(); + + if (!response.getError().isEmpty()) { + log.warn().append("Error in response: ").append(response.getError()).endl(); + } + + // Complete the future - the caller will log the content if needed + future.complete(content); + } else { + log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); + } + } else if (message.hasSetConnectionId()) { + // Client sent connection ID (future use) + log.info().append("Received set_connection_id from client").endl(); + } else if (message.hasTestCommand()) { + // Client sent a test command + String command = message.getTestCommand(); + log.info().append("Received test command from client: ").append(command).endl(); + + if (command.startsWith("TEST:")) { + String resourceName = command.substring(5).trim(); + log.info().append("Client initiated test for resource: ").append(resourceName).endl(); + testRequestResource(resourceName); + } + } else { + log.warn().append("Received unknown message type from client").endl(); + } } catch (InvalidProtocolBufferException e) { - // There is no identifier here, so we cannot properly return an error that is bound to the request. - // Instead, we throw an Exception causing the server to close the entire MessageStream and - // propagate a general error to the client. - throw new RuntimeException(e); + log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); + throw new ObjectCommunicationException("Failed to parse message", e); } } @Override public void onClose() { + // Cancel all pending requests + pendingRequests.values().forEach(future -> future.cancel(true)); + pendingRequests.clear(); + } + + /** + * Request a resource from the client. + * + * @param resourceName the name/path of the resource to request + * @return a future that completes with the resource content, or empty array if not found + */ + public CompletableFuture requestResource(String resourceName) { + String requestId = UUID.randomUUID().toString(); + CompletableFuture future = new CompletableFuture<>(); + pendingRequests.put(requestId, future); + + try { + // Build RemoteFileSourceMetaRequest proto + RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() + .setResourceName(resourceName) + .build(); + + // Wrap in RemoteFileSourceServerRequest (server→client) + RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setMetaRequest(metaRequest) + .build(); + + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); + + log.info().append("Sending resource request for: ").append(resourceName) + .append(" with requestId: ").append(requestId).endl(); + + connection.onData(buffer); + } catch (ObjectCommunicationException e) { + future.completeExceptionally(e); + pendingRequests.remove(requestId); + } + + return future; + } + + /** + * Test method to request a resource and log the result. + * This can be called from the server console to test the bidirectional communication. + * + * @param resourceName the resource to request + */ + public void testRequestResource(String resourceName) { + log.info().append("Testing resource request for: ").append(resourceName).endl(); + requestResource(resourceName) + .orTimeout(30, TimeUnit.SECONDS) + .whenComplete((content, error) -> { + if (error != null) { + log.error().append("Error requesting resource ").append(resourceName) + .append(": ").append(error).endl(); + } else { + log.info().append("Successfully received resource ").append(resourceName) + .append(" (").append(content.length).append(" bytes)").endl(); + if (content.length > 0 && content.length < 1000) { + String contentStr = new String(content, StandardCharsets.UTF_8); + log.info().append("Resource content:\n").append(contentStr).endl(); + } + } + }); } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 65c28479286..5f6aff62a6c 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -11,18 +11,50 @@ option go_package = "github.com/deephaven/deephaven-core/go/internal/proto/remot import "deephaven_core/proto/ticket.proto"; -message RemoteFileSourcePluginRequest { - // A client-specified identifier used to identify the response for this request when multiple requests are in flight. - // This must be set to a unique value. +// Server → Client: Requests sent from server to client via MessageStream +message RemoteFileSourceServerRequest { + // Unique identifier for this request, used to correlate the response string request_id = 1; + + oneof request { + // Request source data/resource from the client + RemoteFileSourceMetaRequest meta_request = 2; + } +} + +// Client → Server: Requests/responses sent from client to server via MessageStream +message RemoteFileSourceClientRequest { + // The request_id from the ServerRequest this is responding to (if applicable) + string request_id = 1; + oneof request { - RemoteFileSourceMetaRequest meta = 2; + // Response to a resource request + RemoteFileSourceMetaResponse meta_response = 2; + + // Future: other client-initiated messages RemoteFileSourceSetConnectionIdRequest set_connection_id = 3; + + // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back + string test_command = 4; } } -// Request meta info from the server plugin +// Request source data/resource from the client message RemoteFileSourceMetaRequest { + // The name/path of the resource being requested (e.g., "com/example/MyClass.java") + string resource_name = 1; +} + +// Response to a resource request +message RemoteFileSourceMetaResponse { + // The content of the resource, or empty if not found + bytes content = 1; + + // Indicates whether the resource was found + bool found = 2; + + // Optional: error message if the resource could not be retrieved + string error = 3; } message RemoteFileSourceSetConnectionIdRequest { @@ -30,7 +62,7 @@ message RemoteFileSourceSetConnectionIdRequest { string connection_id = 1; } -// Fetch the remote file source plugin into the specified ticket +// Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 5de42dfd637..1f07343d979 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -7,10 +7,15 @@ import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ClientData; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ConnectRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamResponse; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -35,6 +40,8 @@ public class JsRemoteFileSourceService extends HasEventHandling { public static final String EVENT_MESSAGE = "message"; @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_CLOSE = "close"; + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") + public static final String EVENT_REQUEST = "request"; private final WorkerConnection connection; private final TypedTicket typedTicket; @@ -135,10 +142,33 @@ private Promise connect() { hasFetched = true; resolve.onInvoke(this); } else { - // Fire message event for subsequent messages - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_MESSAGE, res.getData()); - }, 0); + // Parse the message as RemoteFileSourceServerRequest proto (server→client) + Uint8Array payload = res.getData().getPayload_asU8(); + + try { + RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.deserializeBinary(payload); + + // Check which message type it is + if (message.hasMetaRequest()) { + // Server is requesting a resource from the client + RemoteFileSourceMetaRequest request = message.getMetaRequest(); + + // Fire request event (include request_id from wrapper) + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)); + }, 0); + } else { + // Unknown message type + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_MESSAGE, res.getData()); + }, 0); + } + } catch (Exception e) { + // Failed to parse as proto, fire generic message event + DomGlobal.setTimeout(ignore -> { + fireEvent(EVENT_MESSAGE, res.getData()); + }, 0); + } } }); @@ -177,10 +207,13 @@ public void sendMessage(Object payload) { } StreamRequest req = new StreamRequest(); + ClientData clientData = new ClientData(); // Convert the payload to Uint8Array Uint8Array data; if (payload instanceof String) { + // For now, just convert string to UTF-8 bytes + // In the future, we might want to wrap in a proper proto message data = stringToUtf8((String) payload); } else if (payload instanceof Uint8Array) { data = (Uint8Array) payload; @@ -188,7 +221,33 @@ public void sendMessage(Object payload) { data = Js.uncheckedCast(payload); } - req.getData().setPayload(data); + clientData.setPayload(data); + req.setData(clientData); + messageStream.send(req); + } + + /** + * Test method to verify bidirectional communication. + * Sends a test command to the server, which will request a resource back from the client. + * + * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") + */ + @JsMethod + public void testBidirectionalCommunication(String resourceName) { + if (messageStream == null) { + throw new IllegalStateException("Message stream not connected"); + } + + // Build RemoteFileSourceClientRequest with test_command + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(""); // Empty request_id for test commands + clientRequest.setTestCommand("TEST:" + resourceName); + + // Send via message stream + StreamRequest req = new StreamRequest(); + ClientData clientData = new ClientData(); + clientData.setPayload(clientRequest.serializeBinary()); + req.setData(clientData); messageStream.send(req); } @@ -208,6 +267,75 @@ private void closeStream() { } } + /** + * Event details for a resource request from the server. + * Wraps the proto RemoteFileSourceMetaRequest and provides a respond() method. + */ + @TsInterface + @TsName(namespace = "dh.remotefilesource", name = "ResourceRequest") + public class ResourceRequestEvent { + private final String requestId; + private final RemoteFileSourceMetaRequest protoRequest; + + @JsIgnore + public ResourceRequestEvent(String requestId, RemoteFileSourceMetaRequest protoRequest) { + this.requestId = requestId; + this.protoRequest = protoRequest; + } + + /** + * @return the name/path of the requested resource + */ + @JsProperty + public String getResourceName() { + return protoRequest.getResourceName(); + } + + /** + * Responds to this resource request with the given content. + * + * @param content the resource content (string, ArrayBuffer, or typed array), or null if not found + */ + @JsMethod + public void respond(Object content) { + if (messageStream == null) { + throw new IllegalStateException("Message stream not connected"); + } + + // Build RemoteFileSourceMetaResponse proto + RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); + + if (content == null) { + // Resource not found + response.setFound(false); + response.setContent(new Uint8Array(0)); + } else { + response.setFound(true); + + // Convert content to bytes + if (content instanceof String) { + response.setContent(stringToUtf8((String) content)); + } else if (content instanceof Uint8Array) { + response.setContent((Uint8Array) content); + } else { + response.setContent(Js.uncheckedCast(content)); + } + } + + // Wrap in RemoteFileSourceClientRequest (client→server) + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setMetaResponse(response); + + // Send via message stream + StreamRequest req = new StreamRequest(); + ClientData clientData = new ClientData(); + clientData.setPayload(clientRequest.serializeBinary()); + req.setData(clientData); + messageStream.send(req); + } + } + /** * Wraps a protobuf message in a google.protobuf.Any message. * diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java new file mode 100644 index 00000000000..620652de5da --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -0,0 +1,59 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceClientRequest { + public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( + RemoteFileSourceClientRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceClientRequest message, Object writer); + + public native void clearRequestId(); + + public native void clearMetaResponse(); + + public native void clearSetConnectionId(); + + public native void clearTestCommand(); + + public native String getRequestId(); + + public native RemoteFileSourceMetaResponse getMetaResponse(); + + public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); + + public native String getTestCommand(); + + public native boolean hasMetaResponse(); + + public native boolean hasSetConnectionId(); + + public native boolean hasTestCommand(); + + public native Uint8Array serializeBinary(); + + public native void setRequestId(String value); + + public native void setMetaResponse(); + + public native void setMetaResponse(RemoteFileSourceMetaResponse value); + + public native void setSetConnectionId(); + + public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); + + public native void setTestCommand(String value); +} + diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java index c2d0cc39470..303d1a93cb6 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -20,11 +20,11 @@ public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceMetaRequest message, Object writer); - public static native Object toObject(boolean includeInstance, RemoteFileSourceMetaRequest msg); + public native void clearResourceName(); - public native Uint8Array serializeBinary(); + public native String getResourceName(); - public native Object toObject(); + public native Uint8Array serializeBinary(); - public native Object toObject(boolean includeInstance); + public native void setResourceName(String value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java new file mode 100644 index 00000000000..157c0fad3fb --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java @@ -0,0 +1,43 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceMetaResponse { + public static native RemoteFileSourceMetaResponse deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceMetaResponse deserializeBinaryFromReader( + RemoteFileSourceMetaResponse message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceMetaResponse message, Object writer); + + public native void clearContent(); + + public native void clearFound(); + + public native void clearError(); + + public native Uint8Array getContent(); + + public native boolean getFound(); + + public native String getError(); + + public native Uint8Array serializeBinary(); + + public native void setContent(Uint8Array value); + + public native void setFound(boolean value); + + public native void setError(String value); +} + diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java deleted file mode 100644 index 1acf26d56de..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginRequest.java +++ /dev/null @@ -1,143 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsProperty; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; -import jsinterop.base.JsPropertyMap; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest", - namespace = JsPackage.GLOBAL) -public class RemoteFileSourcePluginRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface SetConnectionIdFieldType { - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - Object getMeta(); - - @JsProperty - String getRequestId(); - - @JsProperty - RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType getSetConnectionId(); - - @JsProperty - void setMeta(Object meta); - - @JsProperty - void setRequestId(String requestId); - - @JsProperty - void setSetConnectionId( - RemoteFileSourcePluginRequest.ToObjectReturnType.SetConnectionIdFieldType setConnectionId); - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType0 { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface SetConnectionIdFieldType { - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - @JsOverlay - static RemoteFileSourcePluginRequest.ToObjectReturnType0 create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - Object getMeta(); - - @JsProperty - String getRequestId(); - - @JsProperty - RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType getSetConnectionId(); - - @JsProperty - void setMeta(Object meta); - - @JsProperty - void setRequestId(String requestId); - - @JsProperty - void setSetConnectionId( - RemoteFileSourcePluginRequest.ToObjectReturnType0.SetConnectionIdFieldType setConnectionId); - } - - public static native RemoteFileSourcePluginRequest deserializeBinary(Uint8Array bytes); - - public static native RemoteFileSourcePluginRequest deserializeBinaryFromReader( - RemoteFileSourcePluginRequest message, Object reader); - - public static native void serializeBinaryToWriter( - RemoteFileSourcePluginRequest message, Object writer); - - public static native RemoteFileSourcePluginRequest.ToObjectReturnType toObject( - boolean includeInstance, RemoteFileSourcePluginRequest msg); - - public native void clearMeta(); - - public native void clearSetConnectionId(); - - public native RemoteFileSourceMetaRequest getMeta(); - - public native int getRequestCase(); - - public native String getRequestId(); - - public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); - - public native boolean hasMeta(); - - public native boolean hasSetConnectionId(); - - public native Uint8Array serializeBinary(); - - public native void setMeta(); - - public native void setMeta(RemoteFileSourceMetaRequest value); - - public native void setRequestId(String value); - - public native void setSetConnectionId(); - - public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); - - public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(); - - public native RemoteFileSourcePluginRequest.ToObjectReturnType0 toObject(boolean includeInstance); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java new file mode 100644 index 00000000000..2df13fcdd17 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -0,0 +1,41 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceServerRequest { + public static native RemoteFileSourceServerRequest deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( + RemoteFileSourceServerRequest message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceServerRequest message, Object writer); + + public native void clearRequestId(); + + public native void clearMetaRequest(); + + public native String getRequestId(); + + public native RemoteFileSourceMetaRequest getMetaRequest(); + + public native boolean hasMetaRequest(); + + public native Uint8Array serializeBinary(); + + public native void setRequestId(String value); + + public native void setMetaRequest(); + + public native void setMetaRequest(RemoteFileSourceMetaRequest value); +} + From 2e61f96cfdbd041d8cc7a2922478e4c3c81dd7be Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 12:12:10 -0600 Subject: [PATCH 09/78] set connection id (#DH-20578) --- .../RemoteFileSourceServicePlugin.java | 32 ++++- .../proto/remotefilesource.proto | 12 ++ .../JsRemoteFileSourceService.java | 123 +++++++++--------- .../RemoteFileSourceServerRequest.java | 10 ++ ...moteFileSourceSetConnectionIdResponse.java | 37 ++++++ 5 files changed, 152 insertions(+), 62 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index df99feeca4f..d577b37eb41 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -11,6 +11,7 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceSetConnectionIdResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -71,11 +72,19 @@ public void testRequestResource(String resourceName) { private static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); + private volatile String connectionId; public RemoteFileSourceMessageStream(final MessageStream connection) { this.connection = connection; } + /** + * @return the connection ID set by the client, or null if not set + */ + public String getConnectionId() { + return connectionId; + } + @Override public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { try { @@ -108,8 +117,27 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); } } else if (message.hasSetConnectionId()) { - // Client sent connection ID (future use) - log.info().append("Received set_connection_id from client").endl(); + // Client sent connection ID + String newConnectionId = message.getSetConnectionId().getConnectionId(); + connectionId = newConnectionId; + log.info().append("Set connection ID from client: ").append(newConnectionId).endl(); + + // Send acknowledgment back to client + RemoteFileSourceSetConnectionIdResponse response = RemoteFileSourceSetConnectionIdResponse.newBuilder() + .setConnectionId(newConnectionId) + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetConnectionIdResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send connection ID acknowledgment: ").append(e).endl(); + } } else if (message.hasTestCommand()) { // Client sent a test command String command = message.getTestCommand(); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 5f6aff62a6c..87ab7a50f3e 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -19,6 +19,9 @@ message RemoteFileSourceServerRequest { oneof request { // Request source data/resource from the client RemoteFileSourceMetaRequest meta_request = 2; + + // Acknowledgment that connection ID was set + RemoteFileSourceSetConnectionIdResponse set_connection_id_response = 3; } } @@ -62,6 +65,15 @@ message RemoteFileSourceSetConnectionIdRequest { string connection_id = 1; } +// Acknowledgment response for setting connection ID +message RemoteFileSourceSetConnectionIdResponse { + // The connection ID that was set + string connection_id = 1; + + // Whether the operation was successful + bool success = 2; +} + // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 1f07343d979..e29b8a1b9a4 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -16,6 +16,8 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -27,6 +29,8 @@ import jsinterop.annotations.JsProperty; import jsinterop.base.Js; +import java.util.HashMap; +import java.util.Map; import java.util.function.Supplier; /** @@ -43,7 +47,6 @@ public class JsRemoteFileSourceService extends HasEventHandling { @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST = "request"; - private final WorkerConnection connection; private final TypedTicket typedTicket; private final Supplier> streamFactory; @@ -51,9 +54,12 @@ public class JsRemoteFileSourceService extends HasEventHandling { private boolean hasFetched; + // Track pending setConnectionId requests + private final Map> pendingSetConnectionIdRequests = new HashMap<>(); + private int requestIdCounter = 0; + @JsIgnore private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { - this.connection = connection; this.typedTicket = typedTicket; this.hasFetched = false; @@ -154,20 +160,26 @@ private Promise connect() { RemoteFileSourceMetaRequest request = message.getMetaRequest(); // Fire request event (include request_id from wrapper) - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); + } else if (message.hasSetConnectionIdResponse()) { + // Server acknowledged connection ID + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetConnectionIdRequests.remove(requestId); + if (resolveCallback != null) { + RemoteFileSourceSetConnectionIdResponse response = message.getSetConnectionIdResponse(); + resolveCallback.onInvoke(response.getSuccess()); + } } else { // Unknown message type - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_MESSAGE, res.getData()); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_MESSAGE, res.getData()), 0); } } catch (Exception e) { // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_MESSAGE, res.getData()); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_MESSAGE, res.getData()), 0); } } }); @@ -176,15 +188,13 @@ private Promise connect() { if (!status.isOk()) { reject.onInvoke(status.getDetails()); } - DomGlobal.setTimeout(ignore -> { - fireEvent(EVENT_CLOSE); - }, 0); + DomGlobal.setTimeout(ignore -> + fireEvent(EVENT_CLOSE), 0); closeStream(); }); - messageStream.onEnd(status -> { - closeStream(); - }); + messageStream.onEnd(status -> + closeStream()); // First message establishes a connection w/ the plugin object instance we're talking to StreamRequest req = new StreamRequest(); @@ -196,54 +206,56 @@ private Promise connect() { } /** - * Sends a message to the server-side plugin. + * Test method to verify bidirectional communication. + * Sends a test command to the server, which will request a resource back from the client. * - * @param payload the message data to send (string, ArrayBuffer, or typed array) + * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") */ @JsMethod - public void sendMessage(Object payload) { - if (messageStream == null) { - throw new IllegalStateException("Message stream not connected"); - } + public void testBidirectionalCommunication(String resourceName) { + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(""); // Empty request_id for test commands + clientRequest.setTestCommand("TEST:" + resourceName); + sendClientRequest(clientRequest); + } - StreamRequest req = new StreamRequest(); - ClientData clientData = new ClientData(); + /** + * Sets the connection ID for this service instance. + * This allows the server to identify and track this specific client connection. + * + * @param connectionId a unique identifier for this connection + * @return a promise that resolves to true if the server successfully set the connection ID, false otherwise + */ + @JsMethod + public Promise setConnectionId(String connectionId) { + return new Promise<>((resolve, reject) -> { + // Generate a unique request ID + String requestId = "setConnectionId-" + (requestIdCounter++); - // Convert the payload to Uint8Array - Uint8Array data; - if (payload instanceof String) { - // For now, just convert string to UTF-8 bytes - // In the future, we might want to wrap in a proper proto message - data = stringToUtf8((String) payload); - } else if (payload instanceof Uint8Array) { - data = (Uint8Array) payload; - } else { - data = Js.uncheckedCast(payload); - } + // Store the resolve callback to call when we get the acknowledgment + pendingSetConnectionIdRequests.put(requestId, resolve); - clientData.setPayload(data); - req.setData(clientData); - messageStream.send(req); + RemoteFileSourceSetConnectionIdRequest setConnIdRequest = new RemoteFileSourceSetConnectionIdRequest(); + setConnIdRequest.setConnectionId(connectionId); + + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setSetConnectionId(setConnIdRequest); + sendClientRequest(clientRequest); + }); } /** - * Test method to verify bidirectional communication. - * Sends a test command to the server, which will request a resource back from the client. + * Helper method to send a RemoteFileSourceClientRequest to the server. * - * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") + * @param clientRequest the client request to send */ - @JsMethod - public void testBidirectionalCommunication(String resourceName) { + @JsIgnore + private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { if (messageStream == null) { throw new IllegalStateException("Message stream not connected"); } - // Build RemoteFileSourceClientRequest with test_command - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(""); // Empty request_id for test commands - clientRequest.setTestCommand("TEST:" + resourceName); - - // Send via message stream StreamRequest req = new StreamRequest(); ClientData clientData = new ClientData(); clientData.setPayload(clientRequest.serializeBinary()); @@ -298,10 +310,6 @@ public String getResourceName() { */ @JsMethod public void respond(Object content) { - if (messageStream == null) { - throw new IllegalStateException("Message stream not connected"); - } - // Build RemoteFileSourceMetaResponse proto RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); @@ -327,12 +335,7 @@ public void respond(Object content) { clientRequest.setRequestId(requestId); clientRequest.setMetaResponse(response); - // Send via message stream - StreamRequest req = new StreamRequest(); - ClientData clientData = new ClientData(); - clientData.setPayload(clientRequest.serializeBinary()); - req.setData(clientData); - messageStream.send(req); + sendClientRequest(clientRequest); } } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 2df13fcdd17..837afccec1d 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -24,12 +24,18 @@ public static native void serializeBinaryToWriter( public native void clearMetaRequest(); + public native void clearSetConnectionIdResponse(); + public native String getRequestId(); public native RemoteFileSourceMetaRequest getMetaRequest(); + public native RemoteFileSourceSetConnectionIdResponse getSetConnectionIdResponse(); + public native boolean hasMetaRequest(); + public native boolean hasSetConnectionIdResponse(); + public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -37,5 +43,9 @@ public static native void serializeBinaryToWriter( public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); + + public native void setSetConnectionIdResponse(); + + public native void setSetConnectionIdResponse(RemoteFileSourceSetConnectionIdResponse value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java new file mode 100644 index 00000000000..82ab25ef314 --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse", + namespace = JsPackage.GLOBAL) +public class RemoteFileSourceSetConnectionIdResponse { + public static native RemoteFileSourceSetConnectionIdResponse deserializeBinary(Uint8Array bytes); + + public static native RemoteFileSourceSetConnectionIdResponse deserializeBinaryFromReader( + RemoteFileSourceSetConnectionIdResponse message, Object reader); + + public static native void serializeBinaryToWriter( + RemoteFileSourceSetConnectionIdResponse message, Object writer); + + public native void clearConnectionId(); + + public native void clearSuccess(); + + public native String getConnectionId(); + + public native boolean getSuccess(); + + public native Uint8Array serializeBinary(); + + public native void setConnectionId(String value); + + public native void setSuccess(boolean value); +} + From f16e317ff773a3d915e02a2f0a5855b626763598 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 12:36:04 -0600 Subject: [PATCH 10/78] Moved clientSessionId to plugin fetch instead of separate message (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 6 +- .../RemoteFileSourceServicePlugin.java | 43 +++++------- .../proto/remotefilesource.proto | 24 ++----- .../deephaven/web/client/api/CoreClient.java | 4 +- .../JsRemoteFileSourceService.java | 49 ++----------- .../RemoteFileSourceClientRequest.java | 9 --- .../RemoteFileSourcePluginFetchRequest.java | 6 ++ .../RemoteFileSourceServerRequest.java | 10 --- ...emoteFileSourceSetConnectionIdRequest.java | 68 ------------------- ...moteFileSourceSetConnectionIdResponse.java | 37 ---------- 10 files changed, 39 insertions(+), 217 deletions(-) delete mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java delete mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index ac2e7b0ba72..bbdd6cdc921 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -63,12 +63,16 @@ public SessionState.ExportObject fetchPlugin(@Nullable final throw new StatusRuntimeException(Status.INVALID_ARGUMENT); } + // Extract optional client session ID from the request (empty string means not provided) + final String clientSessionId = request.getClientSessionId(); + final SessionState.ExportBuilder pluginExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); pluginExportBuilder.require(); final SessionState.ExportObject pluginExport = - pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); + pluginExportBuilder.submit(() -> new RemoteFileSourceServicePlugin( + clientSessionId.isEmpty() ? null : clientSessionId)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index d577b37eb41..452f1aae77d 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -11,7 +11,6 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceSetConnectionIdResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -26,8 +25,18 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); private volatile RemoteFileSourceMessageStream messageStream; + private final String clientSessionId; - public RemoteFileSourceServicePlugin() {} + public RemoteFileSourceServicePlugin() { + this(null); + } + + public RemoteFileSourceServicePlugin(String clientSessionId) { + this.clientSessionId = clientSessionId; + if (clientSessionId != null) { + log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId).endl(); + } + } @Override public String name() { @@ -42,7 +51,7 @@ public boolean isType(Object object) { @Override public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); - messageStream = new RemoteFileSourceMessageStream(connection); + messageStream = new RemoteFileSourceMessageStream(connection, clientSessionId); return messageStream; } @@ -74,8 +83,12 @@ private static class RemoteFileSourceMessageStream implements MessageStream { private final Map> pendingRequests = new ConcurrentHashMap<>(); private volatile String connectionId; - public RemoteFileSourceMessageStream(final MessageStream connection) { + public RemoteFileSourceMessageStream(final MessageStream connection, final String clientSessionId) { this.connection = connection; + this.connectionId = clientSessionId; // Initialize with the ID from the fetch request + if (clientSessionId != null) { + log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ").append(clientSessionId).endl(); + } } /** @@ -116,28 +129,6 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } else { log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); } - } else if (message.hasSetConnectionId()) { - // Client sent connection ID - String newConnectionId = message.getSetConnectionId().getConnectionId(); - connectionId = newConnectionId; - log.info().append("Set connection ID from client: ").append(newConnectionId).endl(); - - // Send acknowledgment back to client - RemoteFileSourceSetConnectionIdResponse response = RemoteFileSourceSetConnectionIdResponse.newBuilder() - .setConnectionId(newConnectionId) - .setSuccess(true) - .build(); - - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setSetConnectionIdResponse(response) - .build(); - - try { - connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); - } catch (ObjectCommunicationException e) { - log.error().append("Failed to send connection ID acknowledgment: ").append(e).endl(); - } } else if (message.hasTestCommand()) { // Client sent a test command String command = message.getTestCommand(); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 87ab7a50f3e..cdfb3e70092 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -19,9 +19,6 @@ message RemoteFileSourceServerRequest { oneof request { // Request source data/resource from the client RemoteFileSourceMetaRequest meta_request = 2; - - // Acknowledgment that connection ID was set - RemoteFileSourceSetConnectionIdResponse set_connection_id_response = 3; } } @@ -34,11 +31,8 @@ message RemoteFileSourceClientRequest { // Response to a resource request RemoteFileSourceMetaResponse meta_response = 2; - // Future: other client-initiated messages - RemoteFileSourceSetConnectionIdRequest set_connection_id = 3; - // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back - string test_command = 4; + string test_command = 3; } } @@ -60,21 +54,11 @@ message RemoteFileSourceMetaResponse { string error = 3; } -message RemoteFileSourceSetConnectionIdRequest { - // The connection ID to set for the remote file source. - string connection_id = 1; -} - -// Acknowledgment response for setting connection ID -message RemoteFileSourceSetConnectionIdResponse { - // The connection ID that was set - string connection_id = 1; - - // Whether the operation was successful - bool success = 2; -} // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; + + // Optional client session ID to identify this client connection + string client_session_id = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 833610bd030..13cf3e79685 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -155,8 +155,8 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public Promise getRemoteFileSourceService() { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + public Promise getRemoteFileSourceService(@JsOptional String clientSessionId) { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get(), clientSessionId); } public Promise getAsIdeConnection() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index e29b8a1b9a4..f621ed157ee 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -16,8 +16,6 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -29,8 +27,6 @@ import jsinterop.annotations.JsProperty; import jsinterop.base.Js; -import java.util.HashMap; -import java.util.Map; import java.util.function.Supplier; /** @@ -54,10 +50,6 @@ public class JsRemoteFileSourceService extends HasEventHandling { private boolean hasFetched; - // Track pending setConnectionId requests - private final Map> pendingSetConnectionIdRequests = new HashMap<>(); - private int requestIdCounter = 0; - @JsIgnore private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { this.typedTicket = typedTicket; @@ -76,16 +68,20 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. * * @param connection the worker connection to use for communication + * @param clientSessionId optional unique identifier for this client session * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public static Promise fetchPlugin(WorkerConnection connection) { + public static Promise fetchPlugin(WorkerConnection connection, String clientSessionId) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); + if (clientSessionId != null && !clientSessionId.isEmpty()) { + fetchRequest.setClientSessionId(clientSessionId); + } // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -162,15 +158,6 @@ private Promise connect() { // Fire request event (include request_id from wrapper) DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); - } else if (message.hasSetConnectionIdResponse()) { - // Server acknowledged connection ID - String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetConnectionIdRequests.remove(requestId); - if (resolveCallback != null) { - RemoteFileSourceSetConnectionIdResponse response = message.getSetConnectionIdResponse(); - resolveCallback.onInvoke(response.getSuccess()); - } } else { // Unknown message type DomGlobal.setTimeout(ignore -> @@ -219,32 +206,6 @@ public void testBidirectionalCommunication(String resourceName) { sendClientRequest(clientRequest); } - /** - * Sets the connection ID for this service instance. - * This allows the server to identify and track this specific client connection. - * - * @param connectionId a unique identifier for this connection - * @return a promise that resolves to true if the server successfully set the connection ID, false otherwise - */ - @JsMethod - public Promise setConnectionId(String connectionId) { - return new Promise<>((resolve, reject) -> { - // Generate a unique request ID - String requestId = "setConnectionId-" + (requestIdCounter++); - - // Store the resolve callback to call when we get the acknowledgment - pendingSetConnectionIdRequests.put(requestId, resolve); - - RemoteFileSourceSetConnectionIdRequest setConnIdRequest = new RemoteFileSourceSetConnectionIdRequest(); - setConnIdRequest.setConnectionId(connectionId); - - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(requestId); - clientRequest.setSetConnectionId(setConnIdRequest); - sendClientRequest(clientRequest); - }); - } - /** * Helper method to send a RemoteFileSourceClientRequest to the server. * diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 620652de5da..28d6736b152 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -24,22 +24,16 @@ public static native void serializeBinaryToWriter( public native void clearMetaResponse(); - public native void clearSetConnectionId(); - public native void clearTestCommand(); public native String getRequestId(); public native RemoteFileSourceMetaResponse getMetaResponse(); - public native RemoteFileSourceSetConnectionIdRequest getSetConnectionId(); - public native String getTestCommand(); public native boolean hasMetaResponse(); - public native boolean hasSetConnectionId(); - public native boolean hasTestCommand(); public native Uint8Array serializeBinary(); @@ -50,9 +44,6 @@ public static native void serializeBinaryToWriter( public native void setMetaResponse(RemoteFileSourceMetaResponse value); - public native void setSetConnectionId(); - - public native void setSetConnectionId(RemoteFileSourceSetConnectionIdRequest value); public native void setTestCommand(String value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index 981a4d40580..203f56eb24e 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -176,8 +176,12 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void clearResultId(); + public native void clearClientSessionId(); + public native Ticket getResultId(); + public native String getClientSessionId(); + public native boolean hasResultId(); public native Uint8Array serializeBinary(); @@ -186,6 +190,8 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void setResultId(Ticket value); + public native void setClientSessionId(String value); + public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject( diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 837afccec1d..2df13fcdd17 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -24,18 +24,12 @@ public static native void serializeBinaryToWriter( public native void clearMetaRequest(); - public native void clearSetConnectionIdResponse(); - public native String getRequestId(); public native RemoteFileSourceMetaRequest getMetaRequest(); - public native RemoteFileSourceSetConnectionIdResponse getSetConnectionIdResponse(); - public native boolean hasMetaRequest(); - public native boolean hasSetConnectionIdResponse(); - public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -43,9 +37,5 @@ public static native void serializeBinaryToWriter( public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); - - public native void setSetConnectionIdResponse(); - - public native void setSetConnectionIdResponse(RemoteFileSourceSetConnectionIdResponse value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java deleted file mode 100644 index 590ba5ee643..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdRequest.java +++ /dev/null @@ -1,68 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsOverlay; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsProperty; -import jsinterop.annotations.JsType; -import jsinterop.base.Js; -import jsinterop.base.JsPropertyMap; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdRequest", - namespace = JsPackage.GLOBAL) -public class RemoteFileSourceSetConnectionIdRequest { - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType { - @JsOverlay - static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) - public interface ToObjectReturnType0 { - @JsOverlay - static RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 create() { - return Js.uncheckedCast(JsPropertyMap.of()); - } - - @JsProperty - String getConnectionId(); - - @JsProperty - void setConnectionId(String connectionId); - } - - public static native RemoteFileSourceSetConnectionIdRequest deserializeBinary(Uint8Array bytes); - - public static native RemoteFileSourceSetConnectionIdRequest deserializeBinaryFromReader( - RemoteFileSourceSetConnectionIdRequest message, Object reader); - - public static native void serializeBinaryToWriter( - RemoteFileSourceSetConnectionIdRequest message, Object writer); - - public static native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType toObject( - boolean includeInstance, RemoteFileSourceSetConnectionIdRequest msg); - - public native String getConnectionId(); - - public native Uint8Array serializeBinary(); - - public native void setConnectionId(String value); - - public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject(); - - public native RemoteFileSourceSetConnectionIdRequest.ToObjectReturnType0 toObject( - boolean includeInstance); -} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java deleted file mode 100644 index 82ab25ef314..00000000000 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceSetConnectionIdResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; - -import elemental2.core.Uint8Array; -import jsinterop.annotations.JsPackage; -import jsinterop.annotations.JsType; - -@JsType( - isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceSetConnectionIdResponse", - namespace = JsPackage.GLOBAL) -public class RemoteFileSourceSetConnectionIdResponse { - public static native RemoteFileSourceSetConnectionIdResponse deserializeBinary(Uint8Array bytes); - - public static native RemoteFileSourceSetConnectionIdResponse deserializeBinaryFromReader( - RemoteFileSourceSetConnectionIdResponse message, Object reader); - - public static native void serializeBinaryToWriter( - RemoteFileSourceSetConnectionIdResponse message, Object writer); - - public native void clearConnectionId(); - - public native void clearSuccess(); - - public native String getConnectionId(); - - public native boolean getSuccess(); - - public native Uint8Array serializeBinary(); - - public native void setConnectionId(String value); - - public native void setSuccess(boolean value); -} - From 101570e01b18b7d128eea1cbedc7a78ddb953ced Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 14:51:48 -0600 Subject: [PATCH 11/78] Cleanup (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 10 +- .../RemoteFileSourceServicePlugin.java | 25 +++-- ...ileSourceTicketResolverFactoryService.java | 3 + .../JsRemoteFileSourceService.java | 100 +++++++++--------- 4 files changed, 76 insertions(+), 62 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index bbdd6cdc921..12276dc5380 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.remotefilesource; import com.google.protobuf.Any; @@ -55,8 +58,8 @@ private static Any parseOrNull(final ByteString data) { } public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, - final Flight.FlightDescriptor descriptor, - final RemoteFileSourcePluginFetchRequest request) { + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { @@ -98,7 +101,8 @@ public SessionState.ExportObject flightInfoFor(@Nullable fina final Any request = parseOrNull(descriptor.getCmd()); if (request == null) { log.error().append("Could not parse remotefilesource command.").endl(); - throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Could not parse remotefilesource command Any."); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "Could not parse remotefilesource command Any."); } if (FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 452f1aae77d..ca971c51d86 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.remotefilesource; import com.google.auto.service.AutoService; @@ -34,7 +37,8 @@ public RemoteFileSourceServicePlugin() { public RemoteFileSourceServicePlugin(String clientSessionId) { this.clientSessionId = clientSessionId; if (clientSessionId != null) { - log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId).endl(); + log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId) + .endl(); } } @@ -49,22 +53,20 @@ public boolean isType(Object object) { } @Override - public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { + public MessageStream compatibleClientConnection(Object object, MessageStream connection) + throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); messageStream = new RemoteFileSourceMessageStream(connection, clientSessionId); return messageStream; } /** - * Test method to trigger a resource request from the server to the client. - * Can be called from the console to test bidirectional communication. - * - * Usage from console: + * Test method to trigger a resource request from the server to the client. Can be called from the console to test + * bidirectional communication. Usage from console: * 
      * service = remote_file_source_service  # The plugin instance
      * service.testRequestResource("com/example/MyClass.java")
      *
- * * @param resourceName the resource to request from the client */ public void testRequestResource(String resourceName) { @@ -81,13 +83,14 @@ public void testRequestResource(String resourceName) { private static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); - private volatile String connectionId; + private final String connectionId; public RemoteFileSourceMessageStream(final MessageStream connection, final String clientSessionId) { this.connection = connection; this.connectionId = clientSessionId; // Initialize with the ID from the fetch request if (clientSessionId != null) { - log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ").append(clientSessionId).endl(); + log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ") + .append(clientSessionId).endl(); } } @@ -193,8 +196,8 @@ public CompletableFuture requestResource(String resourceName) { } /** - * Test method to request a resource and log the result. - * This can be called from the server console to test the bidirectional communication. + * Test method to request a resource and log the result. This can be called from the server console to test the + * bidirectional communication. * * @param resourceName the resource to request */ diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java index 89fa503b9aa..fa398c85d95 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.remotefilesource; import com.google.auto.service.AutoService; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index f621ed157ee..44c48ebce58 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,3 +1,6 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// package io.deephaven.web.client.api.remotefilesource; import com.vertispan.tsdefs.annotations.TsInterface; @@ -88,9 +91,8 @@ public static Promise fetchPlugin(WorkerConnection co // Wrap in google.protobuf.Any with the proper typeUrl Uint8Array anyWrappedBytes = wrapInAny( - "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", - innerRequestBytes - ); + "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", + innerRequestBytes); // Create a FlightDescriptor with the command FlightDescriptor descriptor = new FlightDescriptor(); @@ -98,31 +100,31 @@ public static Promise fetchPlugin(WorkerConnection co descriptor.setCmd(anyWrappedBytes); // Send the getFlightInfo request - return Callbacks.grpcUnaryPromise(c -> - connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply) - ).then(flightInfo -> { - // The first endpoint should contain the ticket for the plugin instance - if (flightInfo.getEndpointList().length > 0) { - // Get the Arrow Flight ticket from the endpoint - io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = - flightInfo.getEndpointList().getAt(0).getTicket(); - - // Convert the Arrow Flight ticket to a Deephaven ticket - Ticket dhTicket = new Ticket(); - dhTicket.setTicket(flightTicket.getTicket_asU8()); - - // Create a TypedTicket for the plugin instance - TypedTicket typedTicket = new TypedTicket(); - typedTicket.setTicket(dhTicket); - typedTicket.setType("RemoteFileSourceService"); - - // Create a new service instance with the typed ticket and connect to it - JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); - return service.connect(); - } else { - return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); - } - }); + return Callbacks.grpcUnaryPromise( + c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)) + .then(flightInfo -> { + // The first endpoint should contain the ticket for the plugin instance + if (flightInfo.getEndpointList().length > 0) { + // Get the Arrow Flight ticket from the endpoint + io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = + flightInfo.getEndpointList().getAt(0).getTicket(); + + // Convert the Arrow Flight ticket to a Deephaven ticket + Ticket dhTicket = new Ticket(); + dhTicket.setTicket(flightTicket.getTicket_asU8()); + + // Create a TypedTicket for the plugin instance + TypedTicket typedTicket = new TypedTicket(); + typedTicket.setTicket(dhTicket); + typedTicket.setType("RemoteFileSourceService"); + + // Create a new service instance with the typed ticket and connect to it + JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); + return service.connect(); + } else { + return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); + } + }); } /** @@ -148,7 +150,8 @@ private Promise connect() { Uint8Array payload = res.getData().getPayload_asU8(); try { - RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.deserializeBinary(payload); + RemoteFileSourceServerRequest message = + RemoteFileSourceServerRequest.deserializeBinary(payload); // Check which message type it is if (message.hasMetaRequest()) { @@ -156,17 +159,15 @@ private Promise connect() { RemoteFileSourceMetaRequest request = message.getMetaRequest(); // Fire request event (include request_id from wrapper) - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, + new ResourceRequestEvent(message.getRequestId(), request)), 0); } else { // Unknown message type - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_MESSAGE, res.getData()), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); } } catch (Exception e) { // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_MESSAGE, res.getData()), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); } } }); @@ -175,13 +176,11 @@ private Promise connect() { if (!status.isOk()) { reject.onInvoke(status.getDetails()); } - DomGlobal.setTimeout(ignore -> - fireEvent(EVENT_CLOSE), 0); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_CLOSE), 0); closeStream(); }); - messageStream.onEnd(status -> - closeStream()); + messageStream.onEnd(status -> closeStream()); // First message establishes a connection w/ the plugin object instance we're talking to StreamRequest req = new StreamRequest(); @@ -193,8 +192,8 @@ private Promise connect() { } /** - * Test method to verify bidirectional communication. - * Sends a test command to the server, which will request a resource back from the client. + * Test method to verify bidirectional communication. Sends a test command to the server, which will request a + * resource back from the client. * * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") */ @@ -241,8 +240,8 @@ private void closeStream() { } /** - * Event details for a resource request from the server. - * Wraps the proto RemoteFileSourceMetaRequest and provides a respond() method. + * Event details for a resource request from the server. Wraps the proto RemoteFileSourceMetaRequest and provides a + * respond() method. */ @TsInterface @TsName(namespace = "dh.remotefilesource", name = "ResourceRequest") @@ -351,11 +350,16 @@ private static Uint8Array stringToUtf8(String str) { } private static int sizeOfVarint(int value) { - if (value < 0) return 10; - if (value < 128) return 1; - if (value < 16384) return 2; - if (value < 2097152) return 3; - if (value < 268435456) return 4; + if (value < 0) + return 10; + if (value < 128) + return 1; + if (value < 16384) + return 2; + if (value < 2097152) + return 3; + if (value < 268435456) + return 4; return 5; } From 30da28267c1d9e0aafd26e3141ab7fa9e5bd7659 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 4 Dec 2025 15:46:05 -0600 Subject: [PATCH 12/78] set execution context (#DH-20578) --- .../RemoteFileSourceServicePlugin.java | 90 ++++++++++++++++++- .../proto/remotefilesource.proto | 26 +++++- .../JsRemoteFileSourceService.java | 55 +++++++++++- .../RemoteFileSourceClientRequest.java | 10 +++ .../RemoteFileSourceServerRequest.java | 10 +++ .../SetExecutionContextRequest.java | 42 +++++++++ .../SetExecutionContextResponse.java | 37 ++++++++ 7 files changed, 266 insertions(+), 4 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index ca971c51d86..2110f6c71c6 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -14,6 +14,7 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -27,6 +28,18 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); + /** + * The execution context ID that identifies the currently active RemoteFileSourceMessageStream. + * This corresponds to the clientSessionId of the most recent script execution. + */ + private static volatile String executionContextId; + + /** + * Top-level package names that should be resolved from the remote source + * (e.g., ["com.example", "org.mycompany"]). + */ + private static volatile java.util.List topLevelPackages = new java.util.ArrayList<>(); + private volatile RemoteFileSourceMessageStream messageStream; private final String clientSessionId; @@ -42,6 +55,57 @@ public RemoteFileSourceServicePlugin(String clientSessionId) { } } + /** + * Gets the current execution context ID. + * + * @return the execution context ID, or null if not set + */ + public static String getExecutionContextId() { + return executionContextId; + } + + /** + * Gets the top-level package names that should be resolved from the remote source. + * + * @return the list of top-level package names + */ + public static java.util.List getTopLevelPackages() { + return new java.util.ArrayList<>(topLevelPackages); + } + + /** + * Sets the execution context ID to identify the currently active RemoteFileSourceMessageStream. + * This should be called when a script execution begins to indicate which client session should + * provide source files. + * + * @param contextId the execution context ID (typically matches a clientSessionId) + * @param packages list of top-level package names to resolve from remote source + */ + public static void setExecutionContextId(String contextId, java.util.List packages) { + executionContextId = contextId; + topLevelPackages = packages != null ? new java.util.ArrayList<>(packages) : new java.util.ArrayList<>(); + log.info().append("Execution context ID set to: ").append(contextId) + .append(" with packages: ").append(String.join(", ", topLevelPackages)).endl(); + } + + /** + * Sets the execution context ID without updating packages (backwards compatibility). + * + * @param contextId the execution context ID (typically matches a clientSessionId) + */ + public static void setExecutionContextId(String contextId) { + setExecutionContextId(contextId, java.util.Collections.emptyList()); + } + + /** + * Clears the execution context ID and top-level packages. + */ + public static void clearExecutionContextId() { + executionContextId = null; + topLevelPackages = new java.util.ArrayList<>(); + log.info().append("Execution context ID cleared").endl(); + } + @Override public String name() { return "RemoteFileSourceService"; @@ -80,7 +144,7 @@ public void testRequestResource(String resourceName) { /** * A message stream for the RemoteFileSourceService. */ - private static class RemoteFileSourceMessageStream implements MessageStream { + public static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); private final String connectionId; @@ -142,6 +206,30 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun log.info().append("Client initiated test for resource: ").append(resourceName).endl(); testRequestResource(resourceName); } + } else if (message.hasSetExecutionContext()) { + // Client is setting the execution context ID + String contextId = message.getSetExecutionContext().getExecutionContextId(); + java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); + setExecutionContextId(contextId, packages); + log.info().append("Client set execution context ID to: ").append(contextId) + .append(" with ").append(packages.size()).append(" top-level packages").endl(); + + // Send acknowledgment back to client + SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() + .setExecutionContextId(contextId) + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetExecutionContextResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); + } } else { log.warn().append("Received unknown message type from client").endl(); } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index cdfb3e70092..70f62e07396 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -19,6 +19,9 @@ message RemoteFileSourceServerRequest { oneof request { // Request source data/resource from the client RemoteFileSourceMetaRequest meta_request = 2; + + // Acknowledgment that execution context was set + SetExecutionContextResponse set_execution_context_response = 3; } } @@ -31,8 +34,11 @@ message RemoteFileSourceClientRequest { // Response to a resource request RemoteFileSourceMetaResponse meta_response = 2; + // Set the execution context ID for script execution + SetExecutionContextRequest set_execution_context = 3; + // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back - string test_command = 3; + string test_command = 4; } } @@ -54,6 +60,24 @@ message RemoteFileSourceMetaResponse { string error = 3; } +// Request to set the execution context ID for script execution +message SetExecutionContextRequest { + // The execution context ID to set (typically matches the clientSessionId) + string execution_context_id = 1; + + // Top-level package names that should be resolved from the remote source + // (e.g., ["com.example", "org.mycompany"]) + repeated string top_level_packages = 2; +} + +// Response acknowledging execution context was set +message SetExecutionContextResponse { + // The execution context ID that was set + string execution_context_id = 1; + + // Whether the operation was successful + bool success = 2; +} // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 44c48ebce58..bd9fc1a824a 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -19,6 +19,8 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; @@ -27,9 +29,12 @@ import io.deephaven.web.client.api.event.HasEventHandling; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; +import jsinterop.annotations.JsOptional; import jsinterop.annotations.JsProperty; import jsinterop.base.Js; +import java.util.HashMap; +import java.util.Map; import java.util.function.Supplier; /** @@ -53,6 +58,10 @@ public class JsRemoteFileSourceService extends HasEventHandling { private boolean hasFetched; + // Track pending setExecutionContext requests + private final Map> pendingSetExecutionContextRequests = new HashMap<>(); + private int requestIdCounter = 0; + @JsIgnore private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { this.typedTicket = typedTicket; @@ -161,6 +170,15 @@ private Promise connect() { // Fire request event (include request_id from wrapper) DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, new ResourceRequestEvent(message.getRequestId(), request)), 0); + } else if (message.hasSetExecutionContextResponse()) { + // Server acknowledged execution context + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetExecutionContextRequests.remove(requestId); + if (resolveCallback != null) { + SetExecutionContextResponse response = message.getSetExecutionContextResponse(); + resolveCallback.onInvoke(response.getSuccess()); + } } else { // Unknown message type DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); @@ -192,8 +210,8 @@ private Promise connect() { } /** - * Test method to verify bidirectional communication. Sends a test command to the server, which will request a - * resource back from the client. + * Test method to verify bidirectional communication. + * Sends a test command to the server, which will request a resource back from the client. * * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") */ @@ -205,6 +223,39 @@ public void testBidirectionalCommunication(String resourceName) { sendClientRequest(clientRequest); } + /** + * Sets the execution context ID on the server to identify which client session should + * provide source files for script execution. + * + * @param executionContextId the execution context ID (typically matches the client session ID) + * @param topLevelPackages array of top-level package names to resolve from remote source (e.g., ["com.example", "org.mycompany"]) + * @return a promise that resolves to true if the server successfully set the execution context, false otherwise + */ + @JsMethod + public Promise setExecutionContext(String executionContextId, @JsOptional String[] topLevelPackages) { + return new Promise<>((resolve, reject) -> { + // Generate a unique request ID + String requestId = "setExecutionContext-" + (requestIdCounter++); + + // Store the resolve callback to call when we get the acknowledgment + pendingSetExecutionContextRequests.put(requestId, resolve); + + SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + setContextRequest.setExecutionContextId(executionContextId); + + if (topLevelPackages != null && topLevelPackages.length > 0) { + for (String pkg : topLevelPackages) { + setContextRequest.addTopLevelPackages(pkg); + } + } + + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setSetExecutionContext(setContextRequest); + sendClientRequest(clientRequest); + }); + } + /** * Helper method to send a RemoteFileSourceClientRequest to the server. * diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 28d6736b152..c6426ab86d5 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -26,16 +26,22 @@ public static native void serializeBinaryToWriter( public native void clearTestCommand(); + public native void clearSetExecutionContext(); + public native String getRequestId(); public native RemoteFileSourceMetaResponse getMetaResponse(); public native String getTestCommand(); + public native SetExecutionContextRequest getSetExecutionContext(); + public native boolean hasMetaResponse(); public native boolean hasTestCommand(); + public native boolean hasSetExecutionContext(); + public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -46,5 +52,9 @@ public static native void serializeBinaryToWriter( public native void setTestCommand(String value); + + public native void setSetExecutionContext(); + + public native void setSetExecutionContext(SetExecutionContextRequest value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 2df13fcdd17..4ce84895ce4 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -24,12 +24,18 @@ public static native void serializeBinaryToWriter( public native void clearMetaRequest(); + public native void clearSetExecutionContextResponse(); + public native String getRequestId(); public native RemoteFileSourceMetaRequest getMetaRequest(); + public native SetExecutionContextResponse getSetExecutionContextResponse(); + public native boolean hasMetaRequest(); + public native boolean hasSetExecutionContextResponse(); + public native Uint8Array serializeBinary(); public native void setRequestId(String value); @@ -37,5 +43,9 @@ public static native void serializeBinaryToWriter( public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); + + public native void setSetExecutionContextResponse(); + + public native void setSetExecutionContextResponse(SetExecutionContextResponse value); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java new file mode 100644 index 00000000000..200472e155f --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -0,0 +1,42 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.JsArray; +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest", + namespace = JsPackage.GLOBAL) +public class SetExecutionContextRequest { + public static native SetExecutionContextRequest deserializeBinary(Uint8Array bytes); + + public static native SetExecutionContextRequest deserializeBinaryFromReader( + SetExecutionContextRequest message, Object reader); + + public static native void serializeBinaryToWriter( + SetExecutionContextRequest message, Object writer); + + public native void clearExecutionContextId(); + + public native void clearTopLevelPackagesList(); + + public native String getExecutionContextId(); + + public native JsArray getTopLevelPackagesList(); + + public native Uint8Array serializeBinary(); + + public native void setExecutionContextId(String value); + + public native void setTopLevelPackagesList(JsArray value); + + public native void addTopLevelPackages(String value); + + public native void addTopLevelPackages(String value, double index); +} + diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java new file mode 100644 index 00000000000..8b43eabcbfb --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; + +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse", + namespace = JsPackage.GLOBAL) +public class SetExecutionContextResponse { + public static native SetExecutionContextResponse deserializeBinary(Uint8Array bytes); + + public static native SetExecutionContextResponse deserializeBinaryFromReader( + SetExecutionContextResponse message, Object reader); + + public static native void serializeBinaryToWriter( + SetExecutionContextResponse message, Object writer); + + public native void clearExecutionContextId(); + + public native void clearSuccess(); + + public native String getExecutionContextId(); + + public native boolean getSuccess(); + + public native Uint8Array serializeBinary(); + + public native void setExecutionContextId(String value); + + public native void setSuccess(boolean value); +} + From 30f201228d8f4af9644acc957cd372c1360553fc Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 5 Dec 2025 10:34:00 -0600 Subject: [PATCH 13/78] Simplified execution context (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 10 +- .../RemoteFileSourceServicePlugin.java | 153 +++++++++--------- .../proto/remotefilesource.proto | 17 +- .../deephaven/web/client/api/CoreClient.java | 4 +- .../JsRemoteFileSourceService.java | 14 +- .../SetExecutionContextRequest.java | 5 - .../SetExecutionContextResponse.java | 5 - 7 files changed, 89 insertions(+), 119 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 12276dc5380..5e23ec3fb5b 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -58,24 +58,20 @@ private static Any parseOrNull(final ByteString data) { } public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, - final Flight.FlightDescriptor descriptor, - final RemoteFileSourcePluginFetchRequest request) { + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { throw new StatusRuntimeException(Status.INVALID_ARGUMENT); } - // Extract optional client session ID from the request (empty string means not provided) - final String clientSessionId = request.getClientSessionId(); - final SessionState.ExportBuilder pluginExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); pluginExportBuilder.require(); final SessionState.ExportObject pluginExport = - pluginExportBuilder.submit(() -> new RemoteFileSourceServicePlugin( - clientSessionId.isEmpty() ? null : clientSessionId)); + pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 2110f6c71c6..2c3133bcabf 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -29,81 +29,48 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); /** - * The execution context ID that identifies the currently active RemoteFileSourceMessageStream. - * This corresponds to the clientSessionId of the most recent script execution. + * The current execution context containing the active message stream and configuration. + * Null when no execution context is active. */ - private static volatile String executionContextId; - - /** - * Top-level package names that should be resolved from the remote source - * (e.g., ["com.example", "org.mycompany"]). - */ - private static volatile java.util.List topLevelPackages = new java.util.ArrayList<>(); + private static volatile RemoteFileSourceExecutionContext executionContext; private volatile RemoteFileSourceMessageStream messageStream; - private final String clientSessionId; public RemoteFileSourceServicePlugin() { - this(null); - } - - public RemoteFileSourceServicePlugin(String clientSessionId) { - this.clientSessionId = clientSessionId; - if (clientSessionId != null) { - log.info().append("RemoteFileSourceServicePlugin created with clientSessionId: ").append(clientSessionId) - .endl(); - } } /** - * Gets the current execution context ID. + * Sets the execution context with the active message stream and top-level packages. + * This should be called when a script execution begins. * - * @return the execution context ID, or null if not set - */ - public static String getExecutionContextId() { - return executionContextId; - } - - /** - * Gets the top-level package names that should be resolved from the remote source. - * - * @return the list of top-level package names - */ - public static java.util.List getTopLevelPackages() { - return new java.util.ArrayList<>(topLevelPackages); - } - - /** - * Sets the execution context ID to identify the currently active RemoteFileSourceMessageStream. - * This should be called when a script execution begins to indicate which client session should - * provide source files. - * - * @param contextId the execution context ID (typically matches a clientSessionId) + * @param messageStream the message stream to set as active (must not be null) * @param packages list of top-level package names to resolve from remote source + * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) */ - public static void setExecutionContextId(String contextId, java.util.List packages) { - executionContextId = contextId; - topLevelPackages = packages != null ? new java.util.ArrayList<>(packages) : new java.util.ArrayList<>(); - log.info().append("Execution context ID set to: ").append(contextId) - .append(" with packages: ").append(String.join(", ", topLevelPackages)).endl(); + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { + if (messageStream == null) { + throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); + } + executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); + log.info().append("Set execution context with ") + .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); } /** - * Sets the execution context ID without updating packages (backwards compatibility). - * - * @param contextId the execution context ID (typically matches a clientSessionId) + * Clears the execution context. */ - public static void setExecutionContextId(String contextId) { - setExecutionContextId(contextId, java.util.Collections.emptyList()); + public static void clearExecutionContext() { + executionContext = null; + log.info().append("Cleared execution context").endl(); } /** - * Clears the execution context ID and top-level packages. + * Gets the current execution context. + * + * @return the execution context */ - public static void clearExecutionContextId() { - executionContextId = null; - topLevelPackages = new java.util.ArrayList<>(); - log.info().append("Execution context ID cleared").endl(); + public static RemoteFileSourceExecutionContext getExecutionContext() { + return executionContext; } @Override @@ -120,7 +87,7 @@ public boolean isType(Object object) { public MessageStream compatibleClientConnection(Object object, MessageStream connection) throws ObjectCommunicationException { connection.onData(ByteBuffer.allocate(0)); - messageStream = new RemoteFileSourceMessageStream(connection, clientSessionId); + messageStream = new RemoteFileSourceMessageStream(connection); return messageStream; } @@ -147,22 +114,9 @@ public void testRequestResource(String resourceName) { public static class RemoteFileSourceMessageStream implements MessageStream { private final MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); - private final String connectionId; - public RemoteFileSourceMessageStream(final MessageStream connection, final String clientSessionId) { + public RemoteFileSourceMessageStream(final MessageStream connection) { this.connection = connection; - this.connectionId = clientSessionId; // Initialize with the ID from the fetch request - if (clientSessionId != null) { - log.info().append("RemoteFileSourceMessageStream initialized with clientSessionId: ") - .append(clientSessionId).endl(); - } - } - - /** - * @return the connection ID set by the client, or null if not set - */ - public String getConnectionId() { - return connectionId; } @Override @@ -207,16 +161,14 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun testRequestResource(resourceName); } } else if (message.hasSetExecutionContext()) { - // Client is setting the execution context ID - String contextId = message.getSetExecutionContext().getExecutionContextId(); + // Client is requesting this message stream to become active java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); - setExecutionContextId(contextId, packages); - log.info().append("Client set execution context ID to: ").append(contextId) - .append(" with ").append(packages.size()).append(" top-level packages").endl(); + setExecutionContext(this, packages); + log.info().append("Client set execution context for this message stream with ") + .append(packages.size()).append(" top-level packages").endl(); // Send acknowledgment back to client SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() - .setExecutionContextId(contextId) .setSuccess(true) .build(); @@ -241,6 +193,12 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun @Override public void onClose() { + // Clear execution context if this was the active stream + RemoteFileSourceExecutionContext context = executionContext; + if (context != null && context.getActiveMessageStream() == this) { + clearExecutionContext(); + } + // Cancel all pending requests pendingRequests.values().forEach(future -> future.cancel(true)); pendingRequests.clear(); @@ -309,4 +267,45 @@ public void testRequestResource(String resourceName) { }); } } + + /** + * Encapsulates the execution context for remote file source operations. + * This includes the currently active message stream and the top-level packages + * that should be resolved from the remote source. + * This class is immutable - a new instance is created each time the context changes. + */ + public static class RemoteFileSourceExecutionContext { + private final RemoteFileSourceMessageStream activeMessageStream; + private final java.util.List topLevelPackages; + + /** + * Creates a new execution context. + * + * @param activeMessageStream the active message stream + * @param topLevelPackages list of top-level package names to resolve from remote source + */ + public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, + java.util.List topLevelPackages) { + this.activeMessageStream = activeMessageStream; + this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); + } + + /** + * Gets the currently active message stream. + * + * @return the active message stream + */ + public RemoteFileSourceMessageStream getActiveMessageStream() { + return activeMessageStream; + } + + /** + * Gets the top-level package names that should be resolved from the remote source. + * + * @return a copy of the list of top-level package names + */ + public java.util.List getTopLevelPackages() { + return new java.util.ArrayList<>(topLevelPackages); + } + } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 70f62e07396..5fef67186f4 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -60,29 +60,20 @@ message RemoteFileSourceMetaResponse { string error = 3; } -// Request to set the execution context ID for script execution +// Request to set the execution context for script execution message SetExecutionContextRequest { - // The execution context ID to set (typically matches the clientSessionId) - string execution_context_id = 1; - // Top-level package names that should be resolved from the remote source - // (e.g., ["com.example", "org.mycompany"]) - repeated string top_level_packages = 2; + // (e.g., ["com", "org"]) + repeated string top_level_packages = 1; } // Response acknowledging execution context was set message SetExecutionContextResponse { - // The execution context ID that was set - string execution_context_id = 1; - // Whether the operation was successful - bool success = 2; + bool success = 1; } // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; - - // Optional client session ID to identify this client connection - string client_session_id = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 13cf3e79685..833610bd030 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -155,8 +155,8 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public Promise getRemoteFileSourceService(@JsOptional String clientSessionId) { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get(), clientSessionId); + public Promise getRemoteFileSourceService() { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); } public Promise getAsIdeConnection() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index bd9fc1a824a..c9be6bc299e 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -80,20 +80,16 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. * * @param connection the worker connection to use for communication - * @param clientSessionId optional unique identifier for this client session * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public static Promise fetchPlugin(WorkerConnection connection, String clientSessionId) { + public static Promise fetchPlugin(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); - if (clientSessionId != null && !clientSessionId.isEmpty()) { - fetchRequest.setClientSessionId(clientSessionId); - } // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -224,15 +220,14 @@ public void testBidirectionalCommunication(String resourceName) { } /** - * Sets the execution context ID on the server to identify which client session should - * provide source files for script execution. + * Sets the execution context on the server to identify this message stream as active + * for script execution. * - * @param executionContextId the execution context ID (typically matches the client session ID) * @param topLevelPackages array of top-level package names to resolve from remote source (e.g., ["com.example", "org.mycompany"]) * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod - public Promise setExecutionContext(String executionContextId, @JsOptional String[] topLevelPackages) { + public Promise setExecutionContext(@JsOptional String[] topLevelPackages) { return new Promise<>((resolve, reject) -> { // Generate a unique request ID String requestId = "setExecutionContext-" + (requestIdCounter++); @@ -241,7 +236,6 @@ public Promise setExecutionContext(String executionContextId, @JsOption pendingSetExecutionContextRequests.put(requestId, resolve); SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); - setContextRequest.setExecutionContextId(executionContextId); if (topLevelPackages != null && topLevelPackages.length > 0) { for (String pkg : topLevelPackages) { diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 200472e155f..3748185297b 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -21,17 +21,12 @@ public static native SetExecutionContextRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextRequest message, Object writer); - public native void clearExecutionContextId(); - public native void clearTopLevelPackagesList(); - public native String getExecutionContextId(); - public native JsArray getTopLevelPackagesList(); public native Uint8Array serializeBinary(); - public native void setExecutionContextId(String value); public native void setTopLevelPackagesList(JsArray value); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java index 8b43eabcbfb..8c27b48e69a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -20,17 +20,12 @@ public static native SetExecutionContextResponse deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextResponse message, Object writer); - public native void clearExecutionContextId(); - public native void clearSuccess(); - public native String getExecutionContextId(); - public native boolean getSuccess(); public native Uint8Array serializeBinary(); - public native void setExecutionContextId(String value); public native void setSuccess(boolean value); } From f44189cb37a75ad53d499691e61acb031eef7f21 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 8 Dec 2025 17:31:06 -0600 Subject: [PATCH 14/78] Basic file sourcing is working (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 119 ++++++++++++++++++ .../engine/util/RemoteFileSourceProvider.java | 37 ++++++ .../engine/util/GroovyDeephavenSession.java | 2 +- .../RemoteFileSourceServicePlugin.java | 82 +++++++++++- 4 files changed, 236 insertions(+), 4 deletions(-) create mode 100644 engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java create mode 100644 engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java new file mode 100644 index 00000000000..b7d65b2bc07 --- /dev/null +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -0,0 +1,119 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; + +/** + * A custom ClassLoader that fetches source files from remote clients via registered RemoteFileSourceProvider instances. + * This is designed to support Groovy script imports where the source files are provided by remote clients. + * + *

When a resource is requested (e.g., for a Groovy import), this class loader: + *

    + *
  1. Checks registered providers to see if they can source the resource
    2. + *
  2. Returns a custom URL with protocol "remotefile://" if a provider can handle it
    3. + *
  3. When that URL is opened, fetches the resource bytes from the provider
    4. + *
+ */ +public class RemoteFileSourceClassLoader extends ClassLoader { + private static final boolean DEBUG = Boolean.getBoolean("RemoteFileSourceClassLoader.debug"); + private static volatile RemoteFileSourceClassLoader instance; + private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); + + public RemoteFileSourceClassLoader(ClassLoader parent) { + super(parent); + instance = this; + } + + public static RemoteFileSourceClassLoader getInstance() { + return instance; + } + + public void registerProvider(RemoteFileSourceProvider provider) { + providers.add(provider); + } + + @Override + protected URL findResource(String name) { + for (RemoteFileSourceProvider provider : providers) { + if (!provider.isActive()) { + continue; + } + try { + Boolean canSource = provider.canSourceResource(name) + .orTimeout(1, TimeUnit.SECONDS) + .get(); + if (Boolean.TRUE.equals(canSource)) { + return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); + } + } catch (Exception e) { + // Continue to next provider + } + } + return super.findResource(name); + } + + /** + * URLStreamHandler that delegates to a RemoteFileSourceProvider to fetch resource bytes. + */ + private static class RemoteFileURLStreamHandler extends URLStreamHandler { + private final RemoteFileSourceProvider provider; + private final String resourceName; + + RemoteFileURLStreamHandler(RemoteFileSourceProvider provider, String resourceName) { + this.provider = provider; + this.resourceName = resourceName; + } + + @Override + protected URLConnection openConnection(URL u) { + return new RemoteFileURLConnection(u, provider, resourceName); + } + } + + /** + * URLConnection that fetches resource bytes from a RemoteFileSourceProvider. + */ + private static class RemoteFileURLConnection extends URLConnection { + private final RemoteFileSourceProvider provider; + private final String resourceName; + private byte[] content; + + RemoteFileURLConnection(URL url, RemoteFileSourceProvider provider, String resourceName) { + super(url); + this.provider = provider; + this.resourceName = resourceName; + } + + @Override + public void connect() throws IOException { + if (!connected) { + try { + content = provider.requestResource(resourceName) + .orTimeout(5, TimeUnit.SECONDS) + .get(); + connected = true; + } catch (Exception e) { + throw new IOException("Failed to fetch remote resource: " + resourceName, e); + } + } + } + + @Override + public InputStream getInputStream() throws IOException { + connect(); + if (content == null || content.length == 0) { + throw new IOException("No content for resource: " + resourceName); + } + return new ByteArrayInputStream(content); + } + } +} diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java new file mode 100644 index 00000000000..4c5408d7833 --- /dev/null +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -0,0 +1,37 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.engine.util; + +import java.util.concurrent.CompletableFuture; + +/** + * Interface for providing remote resources to the ClassLoader. + * Plugins can implement this interface and register with RemoteFileSourceClassLoader + * to provide resources from remote sources. + */ +public interface RemoteFileSourceProvider { + /** + * Check if this provider can source the given resource. + * + * @param resourceName the name of the resource to check (e.g., "com/example/MyClass.groovy") + * @return a CompletableFuture that resolves to true if this provider can handle the resource, false otherwise + */ + CompletableFuture canSourceResource(String resourceName); + + /** + * Request a resource from the remote source. + * + * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy") + * @return a CompletableFuture containing the resource bytes, or null if not found + */ + CompletableFuture requestResource(String resourceName); + + /** + * Check if this provider is currently active and should be used for resource requests. + * + * @return true if this provider is active, false otherwise + */ + boolean isActive(); +} + diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index b581c7be4ef..ebec4bc2977 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -96,7 +96,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession mapping = new ConcurrentHashMap<>(); @Override diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java index 2c3133bcabf..ff733394c5e 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit; @AutoService(ObjectType.class) -public class RemoteFileSourceServicePlugin extends ObjectTypeBase { +public class RemoteFileSourceServicePlugin extends ObjectTypeBase implements io.deephaven.engine.util.RemoteFileSourceProvider { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); /** @@ -37,11 +37,65 @@ public class RemoteFileSourceServicePlugin extends ObjectTypeBase { private volatile RemoteFileSourceMessageStream messageStream; public RemoteFileSourceServicePlugin() { + log.info().append("🎯 RemoteFileSourceServicePlugin constructor called").endl(); + // Register eagerly with the class loader + registerWithClassLoader(); + } + + // RemoteFileSourceProvider interface implementation - delegates to active message stream + + @Override + public CompletableFuture canSourceResource(String resourceName) { + // Only handle .groovy source files, not compiled .class files + if (!resourceName.endsWith(".groovy")) { + return CompletableFuture.completedFuture(false); + } + + RemoteFileSourceExecutionContext context = executionContext; + if (context == null) { + return CompletableFuture.completedFuture(false); + } + + java.util.List topLevelPackages = context.getTopLevelPackages(); + if (topLevelPackages.isEmpty()) { + return CompletableFuture.completedFuture(false); + } + + String resourcePath = resourceName.replace('\\', '/'); + + for (String topLevelPackage : topLevelPackages) { + String packagePath = topLevelPackage.replace('.', '/'); + if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { + log.info().append("✅ Can source: ").append(resourceName).endl(); + return CompletableFuture.completedFuture(true); + } + } + + return CompletableFuture.completedFuture(false); + } + + @Override + public CompletableFuture requestResource(String resourceName) { + log.info().append("📥 Requesting resource: ").append(resourceName).endl(); + + RemoteFileSourceExecutionContext context = executionContext; + if (context == null) { + log.warn().append("No execution context when requesting resource").endl(); + return CompletableFuture.completedFuture(null); + } + + return context.getActiveMessageStream().requestResource(resourceName); + } + + @Override + public boolean isActive() { + return executionContext != null; } /** * Sets the execution context with the active message stream and top-level packages. * This should be called when a script execution begins. + * The plugin (which is registered with the ClassLoader) will route requests to this message stream. * * @param messageStream the message stream to set as active (must not be null) * @param packages list of top-level package names to resolve from remote source @@ -51,6 +105,8 @@ public static void setExecutionContext(RemoteFileSourceMessageStream messageStre if (messageStream == null) { throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); } + + // Set new context - the plugin will automatically route to this message stream executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); log.info().append("Set execution context with ") .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); @@ -60,10 +116,29 @@ public static void setExecutionContext(RemoteFileSourceMessageStream messageStre * Clears the execution context. */ public static void clearExecutionContext() { - executionContext = null; - log.info().append("Cleared execution context").endl(); + if (executionContext != null) { + executionContext = null; + log.info().append("Cleared execution context").endl(); + } + } + + /** + * Register this plugin instance with the RemoteFileSourceClassLoader instance. + * Called once during plugin construction. + */ + private void registerWithClassLoader() { + io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = + io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + + if (classLoader != null) { + classLoader.registerProvider(this); + log.info().append("✅ Registered RemoteFileSourceServicePlugin with RemoteFileSourceClassLoader").endl(); + } else { + log.warn().append("⚠️ RemoteFileSourceClassLoader instance not found - plugin not registered").endl(); + } } + /** * Gets the current execution context. * @@ -241,6 +316,7 @@ public CompletableFuture requestResource(String resourceName) { return future; } + /** * Test method to request a resource and log the result. This can be called from the server console to test the * bidirectional communication. From 0d4334d3391635d54a63111c78ab64ebfd9ec981 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 09:21:38 -0600 Subject: [PATCH 15/78] Comments and cleanup (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 5e23ec3fb5b..f41da7366e1 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -33,22 +33,34 @@ public class RemoteFileSourceCommandResolver implements CommandResolver, WantsTi private static final String FETCH_PLUGIN_TYPE_URL = "type.googleapis.com/" + RemoteFileSourcePluginFetchRequest.getDescriptor().getFullName(); - + /** + * Parses a RemoteFileSourcePluginFetchRequest from the given Any command. + * + * @param command the Any command containing the fetch request + * @return the parsed RemoteFileSourcePluginFetchRequest + * @throws IllegalArgumentException if the command type URL doesn't match the expected fetch plugin type + * @throws UncheckedDeephavenException if the command cannot be parsed as a RemoteFileSourcePluginFetchRequest + */ private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any command) { if (!FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl())) { throw new IllegalArgumentException("Not a valid remotefilesource command: " + command.getTypeUrl()); } - final ByteString bytes = command.getValue(); - final RemoteFileSourcePluginFetchRequest request; try { - request = RemoteFileSourcePluginFetchRequest.parseFrom(bytes); + return RemoteFileSourcePluginFetchRequest.parseFrom(command.getValue()); } catch (InvalidProtocolBufferException e) { throw new UncheckedDeephavenException("Could not parse RemoteFileSourcePluginFetchRequest", e); } - return request; } + /** + * Attempts to parse ByteString data as a protobuf Any message. + * Returns null if parsing fails rather than throwing an exception, allowing callers to handle + * invalid data gracefully. + * + * @param data the ByteString data to parse + * @return the parsed Any message, or null if parsing fails + */ private static Any parseOrNull(final ByteString data) { try { return Any.parseFrom(data); @@ -57,6 +69,17 @@ private static Any parseOrNull(final ByteString data) { } } + /** + * Creates and exports a RemoteFileSourceServicePlugin instance based on the fetch request. + * The plugin is exported to the session using the result ticket specified in the request, + * and flight info is returned containing the endpoint for accessing the plugin. + * + * @param session the session state for the current request + * @param descriptor the flight descriptor containing the command + * @param request the parsed RemoteFileSourcePluginFetchRequest containing the result ticket + * @return a FlightInfo export object containing the plugin endpoint information + * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket + */ public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final RemoteFileSourcePluginFetchRequest request) { @@ -86,6 +109,18 @@ public SessionState.ExportObject fetchPlugin(@Nullable final return SessionState.wrapAsExport(flightInfo); } + /** + * Resolves a flight descriptor to flight info for remote file source commands. + * Handles RemoteFileSourcePluginFetchRequest commands by parsing the descriptor and delegating to the + * appropriate handler method. + * + * @param session the session state for the current request + * @param descriptor the flight descriptor containing the command + * @param logId the log identifier for tracking + * @return a FlightInfo export object for the requested command + * @throws StatusRuntimeException if session is null (UNAUTHENTICATED), the command cannot be parsed, + * or the command type URL is not recognized + */ @Override public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, @@ -156,7 +191,8 @@ public SessionState.ExportBuilder publish(final SessionState session, public SessionState.ExportObject resolve(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final String logId) { - return null; + // use flightInfoFor() instead of resolve() for descriptor handling + throw new UnsupportedOperationException(); } @Override From 6c0ee1a775dcbaf8086a8257a89d66280694d89b Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 09:26:14 -0600 Subject: [PATCH 16/78] Made method private (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index f41da7366e1..32ff1058b9b 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -80,7 +80,7 @@ private static Any parseOrNull(final ByteString data) { * @return a FlightInfo export object containing the plugin endpoint information * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket */ - public SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, + private SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); From db3816c60e0d61fad613406ad783508721525684 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 09:28:35 -0600 Subject: [PATCH 17/78] Made method static (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 32ff1058b9b..4475711928c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -80,7 +80,7 @@ private static Any parseOrNull(final ByteString data) { * @return a FlightInfo export object containing the plugin endpoint information * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket */ - private SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, + private static SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); From bcbe71a9e7fc5072f004f871f482378436ed37b8 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 11:56:32 -0600 Subject: [PATCH 18/78] onResourceRequest method (#DH-20578) --- .../engine/util/RemoteFileSourceClassLoader.java | 2 +- .../JsRemoteFileSourceService.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index b7d65b2bc07..4e8d631d2d5 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -49,7 +49,7 @@ protected URL findResource(String name) { } try { Boolean canSource = provider.canSourceResource(name) - .orTimeout(1, TimeUnit.SECONDS) + .orTimeout(5, TimeUnit.SECONDS) .get(); if (Boolean.TRUE.equals(canSource)) { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index c9be6bc299e..0e7d9ccd889 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -26,7 +26,9 @@ import io.deephaven.web.client.api.Callbacks; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.barrage.stream.BiDiStream; +import io.deephaven.web.client.api.event.EventFn; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.shared.fu.RemoverFn; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsOptional; @@ -268,6 +270,18 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { messageStream.send(req); } + /** + * Registers a listener for resource requests from the server. + * The listener will be called when the server requests a resource from the client. + * + * @param callback the callback to invoke when a resource is requested + * @return a cleanup function that can be called to remove the listener + */ + @JsMethod + public RemoverFn onResourceRequest(EventFn callback) { + return this.addEventListener(EVENT_REQUEST, callback); + } + /** * Closes the message stream connection to the server. */ From 0a5a22331a0325f73594c3b948a521729b739e5f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 10 Dec 2025 15:58:56 -0600 Subject: [PATCH 19/78] JS API types (#DH-20578) --- .../JsRemoteFileSourceService.java | 30 +++++-------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 0e7d9ccd889..0aba6fe8614 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -5,6 +5,7 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; +import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.Uint8Array; import elemental2.dom.DomGlobal; import elemental2.promise.Promise; @@ -26,31 +27,28 @@ import io.deephaven.web.client.api.Callbacks; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.barrage.stream.BiDiStream; -import io.deephaven.web.client.api.event.EventFn; import io.deephaven.web.client.api.event.HasEventHandling; -import io.deephaven.web.shared.fu.RemoverFn; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; +import jsinterop.annotations.JsNullable; import jsinterop.annotations.JsOptional; import jsinterop.annotations.JsProperty; +import jsinterop.annotations.JsType; import jsinterop.base.Js; import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; + /** * JavaScript client for the RemoteFileSource service. Provides bidirectional communication with the server-side * RemoteFileSourceServicePlugin via a message stream. */ -@TsInterface -@TsName(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") +@JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_MESSAGE = "message"; - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_CLOSE = "close"; - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST = "request"; private final TypedTicket typedTicket; @@ -85,7 +83,7 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsMethod - public static Promise fetchPlugin(WorkerConnection connection) { + public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -270,18 +268,6 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { messageStream.send(req); } - /** - * Registers a listener for resource requests from the server. - * The listener will be called when the server requests a resource from the client. - * - * @param callback the callback to invoke when a resource is requested - * @return a cleanup function that can be called to remove the listener - */ - @JsMethod - public RemoverFn onResourceRequest(EventFn callback) { - return this.addEventListener(EVENT_REQUEST, callback); - } - /** * Closes the message stream connection to the server. */ @@ -303,7 +289,7 @@ private void closeStream() { * respond() method. */ @TsInterface - @TsName(namespace = "dh.remotefilesource", name = "ResourceRequest") + @TsName(namespace = "dh.remotefilesource", name = "ResourceRequestEvent") public class ResourceRequestEvent { private final String requestId; private final RemoteFileSourceMetaRequest protoRequest; @@ -328,7 +314,7 @@ public String getResourceName() { * @param content the resource content (string, ArrayBuffer, or typed array), or null if not found */ @JsMethod - public void respond(Object content) { + public void respond(@JsNullable Object content) { // Build RemoteFileSourceMetaResponse proto RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); From 68e762c5e0cf3f7dbd0d6f680e4b0787c310edf8 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 11 Dec 2025 13:41:24 -0600 Subject: [PATCH 20/78] Refactored to use PluginMarker (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 4 + .../RemoteFileSourceCommandResolver.java | 22 +- .../RemoteFileSourceMessageStream.java | 360 ++++++++++++++++ .../RemoteFileSourcePlugin.java | 56 +++ .../RemoteFileSourceServicePlugin.java | 387 ------------------ .../deephaven/plugin/type/PluginMarker.java | 46 +++ .../JsRemoteFileSourceService.java | 8 +- .../RemoteFileSourcePluginFetchRequest.java | 19 +- 8 files changed, 501 insertions(+), 401 deletions(-) create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java create mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java delete mode 100644 plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java create mode 100644 plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 4e8d631d2d5..fa8b5bf03bd 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -41,6 +41,10 @@ public void registerProvider(RemoteFileSourceProvider provider) { providers.add(provider); } + public void unregisterProvider(RemoteFileSourceProvider provider) { + providers.remove(provider); + } + @Override protected URL findResource(String name) { for (RemoteFileSourceProvider provider : providers) { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 4475711928c..af1aabb14db 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -11,6 +11,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import io.deephaven.plugin.type.PluginMarker; import io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest; import io.deephaven.proto.backplane.grpc.Ticket; import io.deephaven.proto.util.Exceptions; @@ -70,9 +71,13 @@ private static Any parseOrNull(final ByteString data) { } /** - * Creates and exports a RemoteFileSourceServicePlugin instance based on the fetch request. - * The plugin is exported to the session using the result ticket specified in the request, - * and flight info is returned containing the endpoint for accessing the plugin. + * Exports a PluginMarker singleton based on the fetch request. + * The marker object is exported to the session using the result ticket specified in the request, + * and flight info is returned containing the endpoint for accessing it. + * + * Note: This exports PluginMarker.INSTANCE as a trusted marker. Plugin-specific routing + * is handled by TypedTicket.type in the ConnectRequest phase, which is validated against + * the plugin's name() method. * * @param session the session state for the current request * @param descriptor the flight descriptor containing the command @@ -86,15 +91,16 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { - throw new StatusRuntimeException(Status.INVALID_ARGUMENT); + throw new StatusRuntimeException(Status.INVALID_ARGUMENT + .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); } - final SessionState.ExportBuilder pluginExportBuilder = + final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); - pluginExportBuilder.require(); + markerExportBuilder.require(); - final SessionState.ExportObject pluginExport = - pluginExportBuilder.submit(RemoteFileSourceServicePlugin::new); + final SessionState.ExportObject markerExport = + markerExportBuilder.submit(() -> PluginMarker.INSTANCE); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java new file mode 100644 index 00000000000..ab5232ae9ee --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -0,0 +1,360 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.remotefilesource; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.plugin.type.ObjectCommunicationException; +import io.deephaven.plugin.type.ObjectType; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +/** + * Message stream implementation for RemoteFileSource bidirectional communication. + * Each instance represents a file source provider for one client connection and implements + * RemoteFileSourceProvider so it can be registered with the ClassLoader. + * Only one MessageStream can be "active" at a time (determined by the execution context). + * The ClassLoader checks isActive() on each registered provider to find the active one. + */ +public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, io.deephaven.engine.util.RemoteFileSourceProvider { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceMessageStream.class); + + /** + * The current execution context containing the active message stream and configuration. + * Null when no execution context is active. + * This is accessed by RemoteFileSourcePlugin.PROVIDER to route resource requests to the + * currently active message stream. + */ + private static volatile RemoteFileSourceExecutionContext executionContext; + + + private final ObjectType.MessageStream connection; + private final Map> pendingRequests = new ConcurrentHashMap<>(); + + public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) { + this.connection = connection; + // Register this instance as a provider with the ClassLoader + registerWithClassLoader(); + } + + // RemoteFileSourceProvider interface implementation - each instance is a provider + + @Override + public java.util.concurrent.CompletableFuture canSourceResource(String resourceName) { + // Only active if this instance is the currently active message stream + if (!isActive()) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + // Only handle .groovy source files, not compiled .class files + if (!resourceName.endsWith(".groovy")) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + RemoteFileSourceExecutionContext context = executionContext; + if (context == null || context.getActiveMessageStream() != this) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + java.util.List topLevelPackages = context.getTopLevelPackages(); + if (topLevelPackages.isEmpty()) { + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + String resourcePath = resourceName.replace('\\', '/'); + + for (String topLevelPackage : topLevelPackages) { + String packagePath = topLevelPackage.replace('.', '/'); + if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { + log.info().append("✅ Can source: ").append(resourceName).endl(); + return java.util.concurrent.CompletableFuture.completedFuture(true); + } + } + + return java.util.concurrent.CompletableFuture.completedFuture(false); + } + + @Override + public java.util.concurrent.CompletableFuture requestResource(String resourceName) { + // Only service requests if this instance is active + if (!isActive()) { + log.warn().append("Request for resource ").append(resourceName) + .append(" on inactive message stream").endl(); + return java.util.concurrent.CompletableFuture.completedFuture(null); + } + + log.info().append("📥 Requesting resource: ").append(resourceName).endl(); + + String requestId = java.util.UUID.randomUUID().toString(); + java.util.concurrent.CompletableFuture future = new java.util.concurrent.CompletableFuture<>(); + pendingRequests.put(requestId, future); + + try { + // Build RemoteFileSourceMetaRequest proto + io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest metaRequest = + io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest.newBuilder() + .setResourceName(resourceName) + .build(); + + // Wrap in RemoteFileSourceServerRequest (server→client) + io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest message = + io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setMetaRequest(metaRequest) + .build(); + + java.nio.ByteBuffer buffer = java.nio.ByteBuffer.wrap(message.toByteArray()); + + log.info().append("Sending resource request for: ").append(resourceName) + .append(" with requestId: ").append(requestId).endl(); + + connection.onData(buffer); + } catch (ObjectCommunicationException e) { + future.completeExceptionally(e); + pendingRequests.remove(requestId); + } + + return future; + } + + @Override + public boolean isActive() { + RemoteFileSourceExecutionContext context = executionContext; + return context != null && context.getActiveMessageStream() == this; + } + + // Static methods for execution context management + + /** + * Sets the execution context with the active message stream and top-level packages. + * This should be called when a script execution begins. + * + * @param messageStream the message stream to set as active (must not be null) + * @param packages list of top-level package names to resolve from remote source + * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) + */ + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { + if (messageStream == null) { + throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); + } + + executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); + log.info().append("Set execution context with ") + .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); + } + + /** + * Clears the execution context. + */ + public static void clearExecutionContext() { + if (executionContext != null) { + executionContext = null; + log.info().append("Cleared execution context").endl(); + } + } + + /** + * Gets the current execution context. + * + * @return the execution context + */ + public static RemoteFileSourceExecutionContext getExecutionContext() { + return executionContext; + } + + // Instance methods for MessageStream implementation + + @Override + public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { + try { + // Parse as RemoteFileSourceClientRequest proto (client→server) + byte[] bytes = new byte[payload.remaining()]; + payload.get(bytes); + RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); + + String requestId = message.getRequestId(); + + if (message.hasMetaResponse()) { + // Client is responding to a resource request + RemoteFileSourceMetaResponse response = message.getMetaResponse(); + + CompletableFuture future = pendingRequests.remove(requestId); + if (future != null) { + byte[] content = response.getContent().toByteArray(); + + log.info().append("Received resource response for requestId: ").append(requestId) + .append(", found: ").append(response.getFound()) + .append(", content length: ").append(content.length).endl(); + + if (!response.getError().isEmpty()) { + log.warn().append("Error in response: ").append(response.getError()).endl(); + } + + future.complete(content); + } else { + log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); + } + } else if (message.hasTestCommand()) { + // Client sent a test command + String command = message.getTestCommand(); + log.info().append("Received test command from client: ").append(command).endl(); + + if (command.startsWith("TEST:")) { + String resourceName = command.substring(5).trim(); + log.info().append("Client initiated test for resource: ").append(resourceName).endl(); + testRequestResource(resourceName); + } + } else if (message.hasSetExecutionContext()) { + // Client is requesting this message stream to become active + java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); + setExecutionContext(this, packages); + log.info().append("Client set execution context for this message stream with ") + .append(packages.size()).append(" top-level packages").endl(); + + // Send acknowledgment back to client + SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetExecutionContextResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); + } + } else { + log.warn().append("Received unknown message type from client").endl(); + } + } catch (InvalidProtocolBufferException e) { + log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); + throw new ObjectCommunicationException("Failed to parse message", e); + } + } + + @Override + public void onClose() { + // Unregister this provider from the ClassLoader + unregisterFromClassLoader(); + + // Clear execution context if this was the active stream + RemoteFileSourceExecutionContext context = executionContext; + if (context != null && context.getActiveMessageStream() == this) { + clearExecutionContext(); + } + + // Cancel all pending requests + pendingRequests.values().forEach(future -> future.cancel(true)); + pendingRequests.clear(); + } + + /** + * Register this message stream instance as a provider with the ClassLoader. + */ + private void registerWithClassLoader() { + io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = + io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + + if (classLoader != null) { + classLoader.registerProvider(this); + log.info().append("✅ Registered RemoteFileSourceMessageStream provider with ClassLoader").endl(); + } else { + log.warn().append("⚠️ RemoteFileSourceClassLoader not available").endl(); + } + } + + /** + * Unregister this message stream instance from the ClassLoader. + */ + private void unregisterFromClassLoader() { + io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = + io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + + if (classLoader != null) { + classLoader.unregisterProvider(this); + log.info().append("🔴 Unregistered RemoteFileSourceMessageStream provider from ClassLoader").endl(); + } + } + + /** + * Test method to request a resource and log the result. This can be called from the server console to test the + * bidirectional communication. + * + * @param resourceName the resource to request + */ + public void testRequestResource(String resourceName) { + log.info().append("Testing resource request for: ").append(resourceName).endl(); + + requestResource(resourceName) + .orTimeout(30, TimeUnit.SECONDS) + .whenComplete((content, error) -> { + if (error != null) { + log.error().append("Error requesting resource ").append(resourceName) + .append(": ").append(error).endl(); + } else { + log.info().append("Successfully received resource ").append(resourceName) + .append(" (").append(content.length).append(" bytes)").endl(); + if (content.length > 0 && content.length < 1000) { + String contentStr = new String(content, StandardCharsets.UTF_8); + log.info().append("Resource content:\n").append(contentStr).endl(); + } + } + }); + } + + /** + * Encapsulates the execution context for remote file source operations. + * This includes the currently active message stream and the top-level packages + * that should be resolved from the remote source. + * This class is immutable - a new instance is created each time the context changes. + */ + public static class RemoteFileSourceExecutionContext { + private final RemoteFileSourceMessageStream activeMessageStream; + private final java.util.List topLevelPackages; + + /** + * Creates a new execution context. + * + * @param activeMessageStream the active message stream + * @param topLevelPackages list of top-level package names to resolve from remote source + */ + public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, + java.util.List topLevelPackages) { + this.activeMessageStream = activeMessageStream; + this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); + } + + /** + * Gets the currently active message stream. + * + * @return the active message stream + */ + public RemoteFileSourceMessageStream getActiveMessageStream() { + return activeMessageStream; + } + + /** + * Gets the top-level package names that should be resolved from the remote source. + * + * @return a copy of the list of top-level package names + */ + public java.util.List getTopLevelPackages() { + return new java.util.ArrayList<>(topLevelPackages); + } + } +} + diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java new file mode 100644 index 00000000000..35d4d4e87bf --- /dev/null +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -0,0 +1,56 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.remotefilesource; + +import com.google.auto.service.AutoService; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.plugin.type.ObjectType; +import io.deephaven.plugin.type.ObjectTypeBase; +import io.deephaven.plugin.type.ObjectCommunicationException; +import io.deephaven.plugin.type.PluginMarker; + +import java.nio.ByteBuffer; + +/** + * ObjectType plugin for RemoteFileSource. This plugin is registered via @AutoService + * and handles creation of RemoteFileSourceMessageStream connections. + * + * This plugin uses a PluginMarker with a type field instead of instanceof checks, + * allowing it to work across language boundaries (Java/Python). The Flight command + * creates a PluginMarker with type="RemoteFileSource" which this plugin recognizes. + * + * Each RemoteFileSourceMessageStream instance registers itself as a provider with the + * ClassLoader when created and unregisters when closed. The ClassLoader checks isActive() + * on each registered provider to find the currently active one. + */ +@AutoService(ObjectType.class) +public class RemoteFileSourcePlugin extends ObjectTypeBase { + private static final Logger log = LoggerFactory.getLogger(RemoteFileSourcePlugin.class); + + @Override + public String name() { + return "GroovyRemoteFileSourcePlugin"; + } + + @Override + public boolean isType(Object object) { + return object instanceof PluginMarker; + } + + @Override + public MessageStream compatibleClientConnection(Object object, MessageStream connection) + throws ObjectCommunicationException { + if (!isType(object)) { + throw new ObjectCommunicationException("Expected RemoteFileSource marker object, got " + object.getClass()); + } + + connection.onData(ByteBuffer.allocate(0)); + + // Create and return a new message stream for this connection + // All the logic is in the static RemoteFileSourceMessageStream class + return new RemoteFileSourceMessageStream(connection); + } +} + diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java deleted file mode 100644 index ff733394c5e..00000000000 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceServicePlugin.java +++ /dev/null @@ -1,387 +0,0 @@ -// -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending -// -package io.deephaven.remotefilesource; - -import com.google.auto.service.AutoService; -import com.google.protobuf.InvalidProtocolBufferException; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; -import io.deephaven.plugin.type.ObjectType; -import io.deephaven.plugin.type.ObjectTypeBase; -import io.deephaven.plugin.type.ObjectCommunicationException; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; -import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; - -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -@AutoService(ObjectType.class) -public class RemoteFileSourceServicePlugin extends ObjectTypeBase implements io.deephaven.engine.util.RemoteFileSourceProvider { - private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceServicePlugin.class); - - /** - * The current execution context containing the active message stream and configuration. - * Null when no execution context is active. - */ - private static volatile RemoteFileSourceExecutionContext executionContext; - - private volatile RemoteFileSourceMessageStream messageStream; - - public RemoteFileSourceServicePlugin() { - log.info().append("🎯 RemoteFileSourceServicePlugin constructor called").endl(); - // Register eagerly with the class loader - registerWithClassLoader(); - } - - // RemoteFileSourceProvider interface implementation - delegates to active message stream - - @Override - public CompletableFuture canSourceResource(String resourceName) { - // Only handle .groovy source files, not compiled .class files - if (!resourceName.endsWith(".groovy")) { - return CompletableFuture.completedFuture(false); - } - - RemoteFileSourceExecutionContext context = executionContext; - if (context == null) { - return CompletableFuture.completedFuture(false); - } - - java.util.List topLevelPackages = context.getTopLevelPackages(); - if (topLevelPackages.isEmpty()) { - return CompletableFuture.completedFuture(false); - } - - String resourcePath = resourceName.replace('\\', '/'); - - for (String topLevelPackage : topLevelPackages) { - String packagePath = topLevelPackage.replace('.', '/'); - if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { - log.info().append("✅ Can source: ").append(resourceName).endl(); - return CompletableFuture.completedFuture(true); - } - } - - return CompletableFuture.completedFuture(false); - } - - @Override - public CompletableFuture requestResource(String resourceName) { - log.info().append("📥 Requesting resource: ").append(resourceName).endl(); - - RemoteFileSourceExecutionContext context = executionContext; - if (context == null) { - log.warn().append("No execution context when requesting resource").endl(); - return CompletableFuture.completedFuture(null); - } - - return context.getActiveMessageStream().requestResource(resourceName); - } - - @Override - public boolean isActive() { - return executionContext != null; - } - - /** - * Sets the execution context with the active message stream and top-level packages. - * This should be called when a script execution begins. - * The plugin (which is registered with the ClassLoader) will route requests to this message stream. - * - * @param messageStream the message stream to set as active (must not be null) - * @param packages list of top-level package names to resolve from remote source - * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) - */ - public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { - if (messageStream == null) { - throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); - } - - // Set new context - the plugin will automatically route to this message stream - executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); - log.info().append("Set execution context with ") - .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); - } - - /** - * Clears the execution context. - */ - public static void clearExecutionContext() { - if (executionContext != null) { - executionContext = null; - log.info().append("Cleared execution context").endl(); - } - } - - /** - * Register this plugin instance with the RemoteFileSourceClassLoader instance. - * Called once during plugin construction. - */ - private void registerWithClassLoader() { - io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = - io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); - - if (classLoader != null) { - classLoader.registerProvider(this); - log.info().append("✅ Registered RemoteFileSourceServicePlugin with RemoteFileSourceClassLoader").endl(); - } else { - log.warn().append("⚠️ RemoteFileSourceClassLoader instance not found - plugin not registered").endl(); - } - } - - - /** - * Gets the current execution context. - * - * @return the execution context - */ - public static RemoteFileSourceExecutionContext getExecutionContext() { - return executionContext; - } - - @Override - public String name() { - return "RemoteFileSourceService"; - } - - @Override - public boolean isType(Object object) { - return object instanceof RemoteFileSourceServicePlugin; - } - - @Override - public MessageStream compatibleClientConnection(Object object, MessageStream connection) - throws ObjectCommunicationException { - connection.onData(ByteBuffer.allocate(0)); - messageStream = new RemoteFileSourceMessageStream(connection); - return messageStream; - } - - /** - * Test method to trigger a resource request from the server to the client. Can be called from the console to test - * bidirectional communication. Usage from console: - * 
-     * service = remote_file_source_service  # The plugin instance
-     * service.testRequestResource("com/example/MyClass.java")
-     *
- * @param resourceName the resource to request from the client - */ - public void testRequestResource(String resourceName) { - if (messageStream == null) { - log.error().append("MessageStream not connected. Please connect a client first.").endl(); - return; - } - messageStream.testRequestResource(resourceName); - } - - /** - * A message stream for the RemoteFileSourceService. - */ - public static class RemoteFileSourceMessageStream implements MessageStream { - private final MessageStream connection; - private final Map> pendingRequests = new ConcurrentHashMap<>(); - - public RemoteFileSourceMessageStream(final MessageStream connection) { - this.connection = connection; - } - - @Override - public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { - try { - // Parse as RemoteFileSourceClientRequest proto (client→server) - byte[] bytes = new byte[payload.remaining()]; - payload.get(bytes); - RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); - - String requestId = message.getRequestId(); - - if (message.hasMetaResponse()) { - // Client is responding to a resource request - RemoteFileSourceMetaResponse response = message.getMetaResponse(); - - CompletableFuture future = pendingRequests.remove(requestId); - if (future != null) { - byte[] content = response.getContent().toByteArray(); - - log.info().append("Received resource response for requestId: ").append(requestId) - .append(", found: ").append(response.getFound()) - .append(", content length: ").append(content.length).endl(); - - if (!response.getError().isEmpty()) { - log.warn().append("Error in response: ").append(response.getError()).endl(); - } - - // Complete the future - the caller will log the content if needed - future.complete(content); - } else { - log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); - } - } else if (message.hasTestCommand()) { - // Client sent a test command - String command = message.getTestCommand(); - log.info().append("Received test command from client: ").append(command).endl(); - - if (command.startsWith("TEST:")) { - String resourceName = command.substring(5).trim(); - log.info().append("Client initiated test for resource: ").append(resourceName).endl(); - testRequestResource(resourceName); - } - } else if (message.hasSetExecutionContext()) { - // Client is requesting this message stream to become active - java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); - setExecutionContext(this, packages); - log.info().append("Client set execution context for this message stream with ") - .append(packages.size()).append(" top-level packages").endl(); - - // Send acknowledgment back to client - SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() - .setSuccess(true) - .build(); - - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setSetExecutionContextResponse(response) - .build(); - - try { - connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); - } catch (ObjectCommunicationException e) { - log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); - } - } else { - log.warn().append("Received unknown message type from client").endl(); - } - } catch (InvalidProtocolBufferException e) { - log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); - throw new ObjectCommunicationException("Failed to parse message", e); - } - } - - @Override - public void onClose() { - // Clear execution context if this was the active stream - RemoteFileSourceExecutionContext context = executionContext; - if (context != null && context.getActiveMessageStream() == this) { - clearExecutionContext(); - } - - // Cancel all pending requests - pendingRequests.values().forEach(future -> future.cancel(true)); - pendingRequests.clear(); - } - - /** - * Request a resource from the client. - * - * @param resourceName the name/path of the resource to request - * @return a future that completes with the resource content, or empty array if not found - */ - public CompletableFuture requestResource(String resourceName) { - String requestId = UUID.randomUUID().toString(); - CompletableFuture future = new CompletableFuture<>(); - pendingRequests.put(requestId, future); - - try { - // Build RemoteFileSourceMetaRequest proto - RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourceName) - .build(); - - // Wrap in RemoteFileSourceServerRequest (server→client) - RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setMetaRequest(metaRequest) - .build(); - - ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); - - log.info().append("Sending resource request for: ").append(resourceName) - .append(" with requestId: ").append(requestId).endl(); - - connection.onData(buffer); - } catch (ObjectCommunicationException e) { - future.completeExceptionally(e); - pendingRequests.remove(requestId); - } - - return future; - } - - - /** - * Test method to request a resource and log the result. This can be called from the server console to test the - * bidirectional communication. - * - * @param resourceName the resource to request - */ - public void testRequestResource(String resourceName) { - log.info().append("Testing resource request for: ").append(resourceName).endl(); - - requestResource(resourceName) - .orTimeout(30, TimeUnit.SECONDS) - .whenComplete((content, error) -> { - if (error != null) { - log.error().append("Error requesting resource ").append(resourceName) - .append(": ").append(error).endl(); - } else { - log.info().append("Successfully received resource ").append(resourceName) - .append(" (").append(content.length).append(" bytes)").endl(); - if (content.length > 0 && content.length < 1000) { - String contentStr = new String(content, StandardCharsets.UTF_8); - log.info().append("Resource content:\n").append(contentStr).endl(); - } - } - }); - } - } - - /** - * Encapsulates the execution context for remote file source operations. - * This includes the currently active message stream and the top-level packages - * that should be resolved from the remote source. - * This class is immutable - a new instance is created each time the context changes. - */ - public static class RemoteFileSourceExecutionContext { - private final RemoteFileSourceMessageStream activeMessageStream; - private final java.util.List topLevelPackages; - - /** - * Creates a new execution context. - * - * @param activeMessageStream the active message stream - * @param topLevelPackages list of top-level package names to resolve from remote source - */ - public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - java.util.List topLevelPackages) { - this.activeMessageStream = activeMessageStream; - this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); - } - - /** - * Gets the currently active message stream. - * - * @return the active message stream - */ - public RemoteFileSourceMessageStream getActiveMessageStream() { - return activeMessageStream; - } - - /** - * Gets the top-level package names that should be resolved from the remote source. - * - * @return a copy of the list of top-level package names - */ - public java.util.List getTopLevelPackages() { - return new java.util.ArrayList<>(topLevelPackages); - } - } -} diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java new file mode 100644 index 00000000000..48d90cdc7d4 --- /dev/null +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -0,0 +1,46 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.plugin.type; + +/** + * A generic marker object for plugin exports that can be shared across multiple plugin types. + * The actual plugin routing is handled by TypedTicket.type, which is validated against + * ObjectType.name() during the ConnectRequest phase. This marker simply indicates "this + * exported object is a plugin placeholder" rather than containing actual plugin-specific data. + */ +public class PluginMarker { + /** + * Static type identifier for PluginMarker objects. + * Uses the fully qualified class name as a standard cross-language identifier. + */ + public static final String TYPE_ID = "io.deephaven.plugin.type.PluginMarker"; + + /** + * Singleton instance for all plugin marker exports. + * Since plugin-specific routing is handled by TypedTicket.type, we don't need + * per-plugin marker instances. + */ + public static final PluginMarker INSTANCE = new PluginMarker(); + + /** + * Private constructor - use INSTANCE singleton. + */ + private PluginMarker() { + } + + /** + * Gets the static type identifier for all PluginMarker objects. + * + * @return the static type identifier + */ + public String getTypeId() { + return TYPE_ID; + } + + @Override + public String toString() { + return "PluginMarker{typeId='" + TYPE_ID + "'}"; + } +} + diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 0aba6fe8614..55e55177293 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -108,7 +108,10 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c return Callbacks.grpcUnaryPromise( c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)) .then(flightInfo -> { - // The first endpoint should contain the ticket for the plugin instance + // The first endpoint contains the ticket for the plugin instance. + // This is the standard Flight pattern: we passed resultTicket in the request, + // the server exported the service to that ticket, and returned a FlightInfo + // with an endpoint containing that same ticket for us to use. if (flightInfo.getEndpointList().length > 0) { // Get the Arrow Flight ticket from the endpoint io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.Ticket flightTicket = @@ -119,9 +122,10 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c dhTicket.setTicket(flightTicket.getTicket_asU8()); // Create a TypedTicket for the plugin instance + // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType("RemoteFileSourceService"); + typedTicket.setType("GroovyRemoteFileSourcePlugin"); // Create a new service instance with the typed ticket and connect to it JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index 203f56eb24e..224a65446f8 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -85,9 +85,15 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); + @JsProperty + String getPluginType(); + @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); + + @JsProperty + void setPluginType(String pluginType); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -158,9 +164,15 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); + @JsProperty + String getPluginType(); + @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); + + @JsProperty + void setPluginType(String pluginType); } public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); @@ -176,11 +188,9 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void clearResultId(); - public native void clearClientSessionId(); - public native Ticket getResultId(); - public native String getClientSessionId(); + public native String getPluginType(); public native boolean hasResultId(); @@ -190,7 +200,8 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void setResultId(Ticket value); - public native void setClientSessionId(String value); + + public native void setPluginType(String value); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); From 2f2c090857f76bb010d5f388d686126d35e66bab Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 11 Dec 2025 16:35:08 -0600 Subject: [PATCH 21/78] Re-using JsWidget for message stream (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- .../RemoteFileSourcePlugin.java | 2 +- .../JsRemoteFileSourceService.java | 153 ++++++------------ 3 files changed, 49 insertions(+), 108 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index af1aabb14db..5f0cfd6556c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -97,7 +97,7 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); - markerExportBuilder.require(); +// markerExportBuilder.require(); final SessionState.ExportObject markerExport = markerExportBuilder.submit(() -> PluginMarker.INSTANCE); diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 35d4d4e87bf..3c3e26048ed 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -31,7 +31,7 @@ public class RemoteFileSourcePlugin extends ObjectTypeBase { @Override public String name() { - return "GroovyRemoteFileSourcePlugin"; + return "DeephavenGroovyRemoteFileSourcePlugin"; } @Override diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 55e55177293..88db3b87808 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -11,10 +11,6 @@ import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ClientData; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.ConnectRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.object_pb.StreamResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; @@ -25,9 +21,11 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; +import io.deephaven.web.client.api.event.Event; import io.deephaven.web.client.api.WorkerConnection; -import io.deephaven.web.client.api.barrage.stream.BiDiStream; import io.deephaven.web.client.api.event.HasEventHandling; +import io.deephaven.web.client.api.widget.JsWidget; +import io.deephaven.web.client.api.widget.WidgetMessageDetails; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; @@ -38,7 +36,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.function.Supplier; /** @@ -48,32 +45,17 @@ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { public static final String EVENT_MESSAGE = "message"; - public static final String EVENT_CLOSE = "close"; public static final String EVENT_REQUEST = "request"; - private final TypedTicket typedTicket; - - private final Supplier> streamFactory; - private BiDiStream messageStream; - - private boolean hasFetched; + private final JsWidget widget; // Track pending setExecutionContext requests private final Map> pendingSetExecutionContextRequests = new HashMap<>(); private int requestIdCounter = 0; @JsIgnore - private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typedTicket) { - this.typedTicket = typedTicket; - this.hasFetched = false; - - // Set up the message stream factory - BiDiStream.Factory factory = connection.streamFactory(); - this.streamFactory = () -> factory.create( - connection.objectServiceClient()::messageStream, - (first, headers) -> connection.objectServiceClient().openMessageStream(first, headers), - (next, headers, c) -> connection.objectServiceClient().nextMessageStream(next, headers, c::apply), - new StreamRequest()); + private JsRemoteFileSourceService(JsWidget widget) { + this.widget = widget; } /** @@ -82,7 +64,7 @@ private JsRemoteFileSourceService(WorkerConnection connection, TypedTicket typed * @param connection the worker connection to use for communication * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ - @JsMethod + @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -125,10 +107,11 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType("GroovyRemoteFileSourcePlugin"); + typedTicket.setType("DeephavenGroovyRemoteFileSourcePlugin"); + + JsWidget widget = new JsWidget(connection, typedTicket); - // Create a new service instance with the typed ticket and connect to it - JsRemoteFileSourceService service = new JsRemoteFileSourceService(connection, typedTicket); + JsRemoteFileSourceService service = new JsRemoteFileSourceService(widget); return service.connect(); } else { return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); @@ -143,70 +126,40 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c */ @JsIgnore private Promise connect() { - if (messageStream != null) { - messageStream.end(); - } - - return new Promise<>((resolve, reject) -> { - messageStream = streamFactory.get(); - - messageStream.onData(res -> { - if (!hasFetched) { - hasFetched = true; - resolve.onInvoke(this); - } else { - // Parse the message as RemoteFileSourceServerRequest proto (server→client) - Uint8Array payload = res.getData().getPayload_asU8(); - - try { - RemoteFileSourceServerRequest message = - RemoteFileSourceServerRequest.deserializeBinary(payload); - - // Check which message type it is - if (message.hasMetaRequest()) { - // Server is requesting a resource from the client - RemoteFileSourceMetaRequest request = message.getMetaRequest(); - - // Fire request event (include request_id from wrapper) - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, - new ResourceRequestEvent(message.getRequestId(), request)), 0); - } else if (message.hasSetExecutionContextResponse()) { - // Server acknowledged execution context - String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetExecutionContextRequests.remove(requestId); - if (resolveCallback != null) { - SetExecutionContextResponse response = message.getSetExecutionContextResponse(); - resolveCallback.onInvoke(response.getSuccess()); - } - } else { - // Unknown message type - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); - } - } catch (Exception e) { - // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, res.getData()), 0); + widget.addEventListener("message", (Event event) -> { + // Parse the message as RemoteFileSourceServerRequest proto (server→client) + Uint8Array payload = event.getDetail().getDataAsU8(); + + try { + RemoteFileSourceServerRequest message = + RemoteFileSourceServerRequest.deserializeBinary(payload); + + if (message.hasMetaRequest()) { + // If server has requested a resource from the client, fire request event + RemoteFileSourceMetaRequest request = message.getMetaRequest(); + + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, + new ResourceRequestEvent(message.getRequestId(), request)), 0); + } else if (message.hasSetExecutionContextResponse()) { + // Server acknowledged execution context was set + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetExecutionContextRequests.remove(requestId); + if (resolveCallback != null) { + SetExecutionContextResponse response = message.getSetExecutionContextResponse(); + resolveCallback.onInvoke(response.getSuccess()); } + } else { + // Unknown message type + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); } - }); - - messageStream.onStatus(status -> { - if (!status.isOk()) { - reject.onInvoke(status.getDetails()); - } - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_CLOSE), 0); - closeStream(); - }); - - messageStream.onEnd(status -> closeStream()); - - // First message establishes a connection w/ the plugin object instance we're talking to - StreamRequest req = new StreamRequest(); - ConnectRequest data = new ConnectRequest(); - data.setSourceId(typedTicket); - req.setConnect(data); - messageStream.send(req); + } catch (Exception e) { + // Failed to parse as proto, fire generic message event + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); + } }); + + return widget.refetch().then(w -> Promise.resolve(this)); } /** @@ -261,15 +214,11 @@ public Promise setExecutionContext(@JsOptional String[] topLevelPackage */ @JsIgnore private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { - if (messageStream == null) { - throw new IllegalStateException("Message stream not connected"); - } + // Serialize the protobuf message to bytes + Uint8Array messageBytes = clientRequest.serializeBinary(); - StreamRequest req = new StreamRequest(); - ClientData clientData = new ClientData(); - clientData.setPayload(clientRequest.serializeBinary()); - req.setData(clientData); - messageStream.send(req); + // Send as Uint8Array (which is an ArrayBufferView, compatible with MessageUnion) + widget.sendMessage(Js.uncheckedCast(messageBytes), null); } /** @@ -277,15 +226,7 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { */ @JsMethod public void close() { - closeStream(); - } - - @JsIgnore - private void closeStream() { - if (messageStream != null) { - messageStream.end(); - messageStream = null; - } + widget.close(); } /** From 9285c58e6fb2dfff40d6c2ea17613ff8753cbcfc Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 11 Dec 2025 17:55:34 -0600 Subject: [PATCH 22/78] Added pluginType field to PluginMarker (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 10 +++- .../RemoteFileSourcePlugin.java | 7 ++- .../deephaven/plugin/type/PluginMarker.java | 48 +++++++++++++++---- .../proto/remotefilesource.proto | 3 ++ .../JsRemoteFileSourceService.java | 5 +- 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 5f0cfd6556c..f86d54d0560 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -95,12 +95,20 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); } + final String pluginType = request.getPluginType(); + if (pluginType == null || pluginType.isEmpty()) { + throw new StatusRuntimeException(Status.INVALID_ARGUMENT + .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_type")); + } + final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); // markerExportBuilder.require(); + // Get singleton marker for this plugin type + // This ensures isType() routing works correctly when multiple plugins use PluginMarker final SessionState.ExportObject markerExport = - markerExportBuilder.submit(() -> PluginMarker.INSTANCE); + markerExportBuilder.submit(() -> PluginMarker.forPluginType(pluginType)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 3c3e26048ed..f76746a0e1c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -36,7 +36,12 @@ public String name() { @Override public boolean isType(Object object) { - return object instanceof PluginMarker; + // We need to check the pluginType + if (object instanceof PluginMarker) { + PluginMarker marker = (PluginMarker) object; + return name().equals(marker.getPluginType()); + } + return false; } @Override diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 48d90cdc7d4..b84ff58dbb9 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -5,9 +5,13 @@ /** * A generic marker object for plugin exports that can be shared across multiple plugin types. - * The actual plugin routing is handled by TypedTicket.type, which is validated against - * ObjectType.name() during the ConnectRequest phase. This marker simply indicates "this - * exported object is a plugin placeholder" rather than containing actual plugin-specific data. + * + * IMPORTANT: The pluginType field is required because ObjectTypeLookup.findObjectType() + * returns the FIRST plugin where isType() returns true. Without plugin-specific identification + * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered + * first would intercept all PluginMarker instances. + * + * This class uses a singleton pattern - one instance per pluginType. */ public class PluginMarker { /** @@ -16,17 +20,31 @@ public class PluginMarker { */ public static final String TYPE_ID = "io.deephaven.plugin.type.PluginMarker"; + private static final java.util.Map INSTANCES = new java.util.concurrent.ConcurrentHashMap<>(); + + private final String pluginType; + /** - * Singleton instance for all plugin marker exports. - * Since plugin-specific routing is handled by TypedTicket.type, we don't need - * per-plugin marker instances. + * Private constructor - use forPluginType() to get singleton instances. + * + * @param pluginType the plugin type identifier (should match the plugin's name() method) */ - public static final PluginMarker INSTANCE = new PluginMarker(); + private PluginMarker(String pluginType) { + this.pluginType = pluginType; + } /** - * Private constructor - use INSTANCE singleton. + * Gets the singleton PluginMarker instance for the specified plugin type. + * + * @param pluginType the plugin type identifier (should match the plugin's name() method) + * @return the singleton PluginMarker for this plugin type + * @throws IllegalArgumentException if pluginType is null or empty */ - private PluginMarker() { + public static PluginMarker forPluginType(String pluginType) { + if (pluginType == null || pluginType.isEmpty()) { + throw new IllegalArgumentException("pluginType cannot be null or empty"); + } + return INSTANCES.computeIfAbsent(pluginType, PluginMarker::new); } /** @@ -38,9 +56,19 @@ public String getTypeId() { return TYPE_ID; } + /** + * Gets the plugin type this marker is intended for. + * This should match the ObjectType.name() of the target plugin. + * + * @return the plugin type identifier + */ + public String getPluginType() { + return pluginType; + } + @Override public String toString() { - return "PluginMarker{typeId='" + TYPE_ID + "'}"; + return "PluginMarker{typeId='" + TYPE_ID + "', pluginType='" + pluginType + "'}"; } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 5fef67186f4..12d9aff1831 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -76,4 +76,7 @@ message SetExecutionContextResponse { // Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; + + // The plugin type identifier to create the PluginMarker for. + string plugin_type = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 88db3b87808..baf9b21c3cb 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -66,12 +66,15 @@ private JsRemoteFileSourceService(JsWidget widget) { */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { + String pluginType = "DeephavenGroovyRemoteFileSourcePlugin"; + // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); + fetchRequest.setPluginType(pluginType); // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -107,7 +110,7 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType("DeephavenGroovyRemoteFileSourcePlugin"); + typedTicket.setType(pluginType); JsWidget widget = new JsWidget(connection, typedTicket); From 8c2f8d13cfcfed45d638ff448d03790038799a80 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 12 Dec 2025 12:52:14 -0600 Subject: [PATCH 23/78] Removed redundant field and renamed pluginType to pluginName (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 11 ++-- .../RemoteFileSourcePlugin.java | 3 +- .../deephaven/plugin/type/PluginMarker.java | 55 +++++++------------ .../proto/remotefilesource.proto | 4 +- .../JsRemoteFileSourceService.java | 6 +- .../RemoteFileSourcePluginFetchRequest.java | 13 ++--- 6 files changed, 37 insertions(+), 55 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index f86d54d0560..c1d973609af 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -95,20 +95,19 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); } - final String pluginType = request.getPluginType(); - if (pluginType == null || pluginType.isEmpty()) { + final String pluginName = request.getPluginName(); + if (pluginName.isEmpty()) { throw new StatusRuntimeException(Status.INVALID_ARGUMENT - .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_type")); + .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_name")); } final SessionState.ExportBuilder markerExportBuilder = session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); -// markerExportBuilder.require(); - // Get singleton marker for this plugin type + // Get singleton marker for this plugin name // This ensures isType() routing works correctly when multiple plugins use PluginMarker final SessionState.ExportObject markerExport = - markerExportBuilder.submit(() -> PluginMarker.forPluginType(pluginType)); + markerExportBuilder.submit(() -> PluginMarker.forPluginName(pluginName)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index f76746a0e1c..09af2b3430f 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -36,10 +36,9 @@ public String name() { @Override public boolean isType(Object object) { - // We need to check the pluginType if (object instanceof PluginMarker) { PluginMarker marker = (PluginMarker) object; - return name().equals(marker.getPluginType()); + return name().equals(marker.getPluginName()); } return false; } diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index b84ff58dbb9..be32f2227fb 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -6,69 +6,54 @@ /** * A generic marker object for plugin exports that can be shared across multiple plugin types. * - * IMPORTANT: The pluginType field is required because ObjectTypeLookup.findObjectType() + * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. * - * This class uses a singleton pattern - one instance per pluginType. + * This class uses a singleton pattern - one instance per pluginName. */ public class PluginMarker { - /** - * Static type identifier for PluginMarker objects. - * Uses the fully qualified class name as a standard cross-language identifier. - */ - public static final String TYPE_ID = "io.deephaven.plugin.type.PluginMarker"; - private static final java.util.Map INSTANCES = new java.util.concurrent.ConcurrentHashMap<>(); - private final String pluginType; + private final String pluginName; /** - * Private constructor - use forPluginType() to get singleton instances. + * Private constructor - use forPluginName() to get singleton instances. * - * @param pluginType the plugin type identifier (should match the plugin's name() method) + * @param pluginName the plugin name identifier (should match the plugin's name() method) */ - private PluginMarker(String pluginType) { - this.pluginType = pluginType; + private PluginMarker(String pluginName) { + this.pluginName = pluginName; } /** - * Gets the singleton PluginMarker instance for the specified plugin type. + * Gets the singleton PluginMarker instance for the specified plugin name. * - * @param pluginType the plugin type identifier (should match the plugin's name() method) - * @return the singleton PluginMarker for this plugin type - * @throws IllegalArgumentException if pluginType is null or empty + * @param pluginName the plugin name identifier (should match the plugin's name() method) + * @return the singleton PluginMarker for this plugin name + * @throws IllegalArgumentException if pluginName is null or empty */ - public static PluginMarker forPluginType(String pluginType) { - if (pluginType == null || pluginType.isEmpty()) { - throw new IllegalArgumentException("pluginType cannot be null or empty"); + public static PluginMarker forPluginName(String pluginName) { + if (pluginName == null || pluginName.isEmpty()) { + throw new IllegalArgumentException("pluginName cannot be null or empty"); } - return INSTANCES.computeIfAbsent(pluginType, PluginMarker::new); - } - - /** - * Gets the static type identifier for all PluginMarker objects. - * - * @return the static type identifier - */ - public String getTypeId() { - return TYPE_ID; + return INSTANCES.computeIfAbsent(pluginName, PluginMarker::new); } /** - * Gets the plugin type this marker is intended for. + * Gets the plugin name this marker is intended for. * This should match the ObjectType.name() of the target plugin. * - * @return the plugin type identifier + * @return the plugin name identifier */ - public String getPluginType() { - return pluginType; + public String getPluginName() { + return pluginName; } @Override public String toString() { - return "PluginMarker{typeId='" + TYPE_ID + "', pluginType='" + pluginType + "'}"; + return "PluginMarker{pluginName='" + pluginName + "'}"; } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 12d9aff1831..177ff1bde41 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -77,6 +77,6 @@ message SetExecutionContextResponse { message RemoteFileSourcePluginFetchRequest { io.deephaven.proto.backplane.grpc.Ticket result_id = 1; - // The plugin type identifier to create the PluginMarker for. - string plugin_type = 2; + // The plugin name to create the PluginMarker for. + string plugin_name = 2; } \ No newline at end of file diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index baf9b21c3cb..2896583e86e 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -66,7 +66,7 @@ private JsRemoteFileSourceService(JsWidget widget) { */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginType = "DeephavenGroovyRemoteFileSourcePlugin"; + String pluginName = "DeephavenPythonRemoteFileSourcePlugin"; // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -74,7 +74,7 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); - fetchRequest.setPluginType(pluginType); + fetchRequest.setPluginName(pluginName); // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -110,7 +110,7 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType(pluginType); + typedTicket.setType(pluginName); JsWidget widget = new JsWidget(connection, typedTicket); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index 224a65446f8..bd48a2068e5 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -86,14 +86,14 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); @JsProperty - String getPluginType(); + String getPluginName(); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); @JsProperty - void setPluginType(String pluginType); + void setPluginName(String pluginName); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -165,14 +165,14 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); @JsProperty - String getPluginType(); + String getPluginName(); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); @JsProperty - void setPluginType(String pluginType); + void setPluginName(String pluginName); } public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); @@ -190,7 +190,7 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native Ticket getResultId(); - public native String getPluginType(); + public native String getPluginName(); public native boolean hasResultId(); @@ -200,8 +200,7 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void setResultId(Ticket value); - - public native void setPluginType(String value); + public native void setPluginName(String value); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); From f21ced838a4d2c75784dbd6f6eed07b9dab494b0 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 12 Dec 2025 14:17:28 -0600 Subject: [PATCH 24/78] Fixed incorrect plugin name (#DH-20578) --- .../client/api/remotefilesource/JsRemoteFileSourceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 2896583e86e..e825a842ef9 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -66,7 +66,7 @@ private JsRemoteFileSourceService(JsWidget widget) { */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenPythonRemoteFileSourcePlugin"; + String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); From 47b154a3ebc9486178b2dfc00400d634a4b4177c Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 16 Dec 2025 11:22:42 -0600 Subject: [PATCH 25/78] Cleanup (#DH-20578) --- .../deephaven/plugin/type/PluginMarker.java | 2 -- .../JsRemoteFileSourceService.java | 26 +++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index be32f2227fb..570af422ecc 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -5,12 +5,10 @@ /** * A generic marker object for plugin exports that can be shared across multiple plugin types. - * * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. - * * This class uses a singleton pattern - one instance per pluginName. */ public class PluginMarker { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index e825a842ef9..8244d6da205 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -59,15 +59,14 @@ private JsRemoteFileSourceService(JsWidget widget) { } /** - * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. + * Fetches the FlightInfo for the plugin fetch command. * - * @param connection the worker connection to use for communication - * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream + * @param connection the worker connection to use + * @param pluginName the name of the plugin to fetch + * @return a promise that resolves to the FlightInfo for the plugin fetch */ @JsIgnore - public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; - + private static Promise fetchPluginFlightInfo(WorkerConnection connection, String pluginName) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -91,7 +90,19 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // Send the getFlightInfo request return Callbacks.grpcUnaryPromise( - c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)) + c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); + } + + /** + * Fetches a RemoteFileSource plugin instance from the server and establishes a message stream connection. + * + * @param connection the worker connection to use for communication + * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream + */ + @JsIgnore + public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { + String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; + return fetchPluginFlightInfo(connection, pluginName) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. // This is the standard Flight pattern: we passed resultTicket in the request, @@ -365,4 +376,3 @@ private static int writeVarint(Uint8Array buffer, int pos, int value) { return pos; } } - From 46716337e6c1bd749526771533728ab1123e8d99 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 17 Dec 2025 15:45:54 -0600 Subject: [PATCH 26/78] Renamed event (#DH-20578) --- .../io/deephaven/engine/util/RemoteFileSourceClassLoader.java | 1 + .../api/remotefilesource/JsRemoteFileSourceService.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index fa8b5bf03bd..a3235ad9450 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -55,6 +55,7 @@ protected URL findResource(String name) { Boolean canSource = provider.canSourceResource(name) .orTimeout(5, TimeUnit.SECONDS) .get(); + if (Boolean.TRUE.equals(canSource)) { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 8244d6da205..ba4367ecb0d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -45,7 +45,7 @@ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { public static final String EVENT_MESSAGE = "message"; - public static final String EVENT_REQUEST = "request"; + public static final String EVENT_REQUEST_SOURCE = "requestsource"; private final JsWidget widget; @@ -152,7 +152,7 @@ private Promise connect() { // If server has requested a resource from the client, fire request event RemoteFileSourceMetaRequest request = message.getMetaRequest(); - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST, + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, new ResourceRequestEvent(message.getRequestId(), request)), 0); } else if (message.hasSetExecutionContextResponse()) { // Server acknowledged execution context was set From aaec3da4a2b783619b5d53c7c483f492a5435fb2 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 18 Dec 2025 15:17:44 -0600 Subject: [PATCH 27/78] Passing in relative file paths instead of top-level folder (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 40 +++++++++---------- .../proto/remotefilesource.proto | 6 +-- .../JsRemoteFileSourceService.java | 37 +++++++++++------ .../SetExecutionContextRequest.java | 11 +++-- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index ab5232ae9ee..1067966d2c0 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -67,16 +67,14 @@ public java.util.concurrent.CompletableFuture canSourceResource(String return java.util.concurrent.CompletableFuture.completedFuture(false); } - java.util.List topLevelPackages = context.getTopLevelPackages(); - if (topLevelPackages.isEmpty()) { + java.util.List resourcePaths = context.getResourcePaths(); + if (resourcePaths.isEmpty()) { return java.util.concurrent.CompletableFuture.completedFuture(false); } - String resourcePath = resourceName.replace('\\', '/'); - - for (String topLevelPackage : topLevelPackages) { - String packagePath = topLevelPackage.replace('.', '/'); - if (resourcePath.startsWith(packagePath + "/") || resourcePath.startsWith(packagePath)) { + // Resource names from ClassLoader always use forward slashes, not backslashes + for (String contextResourcePath : resourcePaths) { + if (resourceName.equals(contextResourcePath)) { log.info().append("✅ Can source: ").append(resourceName).endl(); return java.util.concurrent.CompletableFuture.completedFuture(true); } @@ -137,11 +135,11 @@ public boolean isActive() { // Static methods for execution context management /** - * Sets the execution context with the active message stream and top-level packages. + * Sets the execution context with the active message stream and resource paths. * This should be called when a script execution begins. * * @param messageStream the message stream to set as active (must not be null) - * @param packages list of top-level package names to resolve from remote source + * @param packages list of resource paths to resolve from remote source * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) */ public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { @@ -151,7 +149,7 @@ public static void setExecutionContext(RemoteFileSourceMessageStream messageStre executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); log.info().append("Set execution context with ") - .append(packages != null ? packages.size() : 0).append(" top-level packages").endl(); + .append(packages != null ? packages.size() : 0).append(" resource paths").endl(); } /** @@ -217,10 +215,10 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } } else if (message.hasSetExecutionContext()) { // Client is requesting this message stream to become active - java.util.List packages = message.getSetExecutionContext().getTopLevelPackagesList(); + java.util.List packages = message.getSetExecutionContext().getResourcePathsList(); setExecutionContext(this, packages); log.info().append("Client set execution context for this message stream with ") - .append(packages.size()).append(" top-level packages").endl(); + .append(packages.size()).append(" resource paths").endl(); // Send acknowledgment back to client SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() @@ -318,24 +316,24 @@ public void testRequestResource(String resourceName) { /** * Encapsulates the execution context for remote file source operations. - * This includes the currently active message stream and the top-level packages + * This includes the currently active message stream and the resource paths * that should be resolved from the remote source. * This class is immutable - a new instance is created each time the context changes. */ public static class RemoteFileSourceExecutionContext { private final RemoteFileSourceMessageStream activeMessageStream; - private final java.util.List topLevelPackages; + private final java.util.List resourcePaths; /** * Creates a new execution context. * * @param activeMessageStream the active message stream - * @param topLevelPackages list of top-level package names to resolve from remote source + * @param resourcePaths list of resource paths to resolve from remote source */ public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - java.util.List topLevelPackages) { + java.util.List resourcePaths) { this.activeMessageStream = activeMessageStream; - this.topLevelPackages = topLevelPackages != null ? topLevelPackages : java.util.Collections.emptyList(); + this.resourcePaths = resourcePaths != null ? resourcePaths : java.util.Collections.emptyList(); } /** @@ -348,12 +346,12 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { } /** - * Gets the top-level package names that should be resolved from the remote source. + * Gets the resource paths that should be resolved from the remote source. * - * @return a copy of the list of top-level package names + * @return a copy of the list of resource paths */ - public java.util.List getTopLevelPackages() { - return new java.util.ArrayList<>(topLevelPackages); + public java.util.List getResourcePaths() { + return new java.util.ArrayList<>(resourcePaths); } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 177ff1bde41..b383a776e6b 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -62,9 +62,9 @@ message RemoteFileSourceMetaResponse { // Request to set the execution context for script execution message SetExecutionContextRequest { - // Top-level package names that should be resolved from the remote source - // (e.g., ["com", "org"]) - repeated string top_level_packages = 1; + // Resource paths that should be resolved from the remote source + // (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) + repeated string resource_paths = 1; } // Response acknowledging execution context was set diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index ba4367ecb0d..c91ce77fefa 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -33,6 +33,7 @@ import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; import jsinterop.base.Js; +import org.jetbrains.annotations.NotNull; import java.util.HashMap; import java.util.Map; @@ -194,11 +195,11 @@ public void testBidirectionalCommunication(String resourceName) { * Sets the execution context on the server to identify this message stream as active * for script execution. * - * @param topLevelPackages array of top-level package names to resolve from remote source (e.g., ["com.example", "org.mycompany"]) + * @param resourcePaths array of resource paths to resolve from remote source (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod - public Promise setExecutionContext(@JsOptional String[] topLevelPackages) { + public Promise setExecutionContext(@JsOptional String[] resourcePaths) { return new Promise<>((resolve, reject) -> { // Generate a unique request ID String requestId = "setExecutionContext-" + (requestIdCounter++); @@ -206,19 +207,31 @@ public Promise setExecutionContext(@JsOptional String[] topLevelPackage // Store the resolve callback to call when we get the acknowledgment pendingSetExecutionContextRequests.put(requestId, resolve); - SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); + sendClientRequest(clientRequest); + }); + } - if (topLevelPackages != null && topLevelPackages.length > 0) { - for (String pkg : topLevelPackages) { - setContextRequest.addTopLevelPackages(pkg); - } + /** + * Helper method to build a RemoteFileSourceClientRequest for setting execution context. + * + * @param resourcePaths array of resource paths to resolve + * @param requestId unique request ID + * @return the constructed RemoteFileSourceClientRequest + */ + private static @NotNull RemoteFileSourceClientRequest getSetExecutionContextRequest(String[] resourcePaths, String requestId) { + SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + + if (resourcePaths != null) { + for (String resourcePath : resourcePaths) { + setContextRequest.addResourcePaths(resourcePath); } + } - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(requestId); - clientRequest.setSetExecutionContext(setContextRequest); - sendClientRequest(clientRequest); - }); + RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + clientRequest.setRequestId(requestId); + clientRequest.setSetExecutionContext(setContextRequest); + return clientRequest; } /** diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 3748185297b..8967670152f 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -21,17 +21,16 @@ public static native SetExecutionContextRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextRequest message, Object writer); - public native void clearTopLevelPackagesList(); + public native void clearResourcePathsList(); - public native JsArray getTopLevelPackagesList(); + public native JsArray getResourcePathsList(); public native Uint8Array serializeBinary(); + public native void setResourcePathsList(JsArray value); - public native void setTopLevelPackagesList(JsArray value); + public native String addResourcePaths(String value); - public native void addTopLevelPackages(String value); - - public native void addTopLevelPackages(String value, double index); + public native String addResourcePaths(String value, double index); } From dfc7a2ffedfee499ae6f16451cbad7b82f65f193 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 18 Dec 2025 17:20:49 -0600 Subject: [PATCH 28/78] Added remotefilesource plugin to server build.gradle (#DH-20578) --- server/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/server/build.gradle b/server/build.gradle index f59344a3cdf..564093a1f3b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -93,6 +93,7 @@ dependencies { runtimeOnly project(':plugin-figure') runtimeOnly project(':plugin-partitionedtable') runtimeOnly project(':plugin-hierarchicaltable') + runtimeOnly project(':plugin-remotefilesource') implementation project(':plugin-gc-app') api platform(libs.grpc.bom) From a3f2c72af1bab5db9cc11d11a84599728229ea59 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 19 Dec 2025 13:56:20 -0600 Subject: [PATCH 29/78] Regenerate bindings --- .../RemoteFileSourceClientRequest.java | 280 +++++++++++++++++- .../RemoteFileSourceMetaRequest.java | 39 ++- .../RemoteFileSourceMetaResponse.java | 228 +++++++++++++- .../RemoteFileSourcePluginFetchRequest.java | 24 +- .../RemoteFileSourceServerRequest.java | 134 ++++++++- .../SetExecutionContextRequest.java | 59 +++- .../SetExecutionContextResponse.java | 41 ++- .../RequestCase.java | 15 + .../RequestCase.java | 8 +- 9 files changed, 777 insertions(+), 51 deletions(-) create mode 100644 web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java rename web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/{remotefilesourcepluginrequest => remotefilesourceserverrequest}/RequestCase.java (65%) diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index c6426ab86d5..314b642900a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -3,15 +3,266 @@ // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; +import elemental2.core.JsArray; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest", namespace = JsPackage.GLOBAL) public class RemoteFileSourceClientRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaResponseFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent( + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextFieldType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType getMetaResponse(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType getSetExecutionContext(); + + @JsProperty + String getTestCommand(); + + @JsProperty + void setMetaResponse( + RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType metaResponse); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContext( + RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType setExecutionContext); + + @JsProperty + void setTestCommand(String testCommand); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaResponseFieldType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType of( + Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent( + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextFieldType { + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + + @JsOverlay + static RemoteFileSourceClientRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType getMetaResponse(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType getSetExecutionContext(); + + @JsProperty + String getTestCommand(); + + @JsProperty + void setMetaResponse( + RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType metaResponse); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContext( + RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType setExecutionContext); + + @JsProperty + void setTestCommand(String testCommand); + } + public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( @@ -20,41 +271,46 @@ public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceClientRequest message, Object writer); - public native void clearRequestId(); + public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceClientRequest msg); public native void clearMetaResponse(); - public native void clearTestCommand(); - public native void clearSetExecutionContext(); - public native String getRequestId(); + public native void clearTestCommand(); public native RemoteFileSourceMetaResponse getMetaResponse(); - public native String getTestCommand(); + public native int getRequestCase(); + + public native String getRequestId(); public native SetExecutionContextRequest getSetExecutionContext(); - public native boolean hasMetaResponse(); + public native String getTestCommand(); - public native boolean hasTestCommand(); + public native boolean hasMetaResponse(); public native boolean hasSetExecutionContext(); - public native Uint8Array serializeBinary(); + public native boolean hasTestCommand(); - public native void setRequestId(String value); + public native Uint8Array serializeBinary(); public native void setMetaResponse(); public native void setMetaResponse(RemoteFileSourceMetaResponse value); - - public native void setTestCommand(String value); + public native void setRequestId(String value); public native void setSetExecutionContext(); public native void setSetExecutionContext(SetExecutionContextRequest value); -} + public native void setTestCommand(String value); + + public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java index 303d1a93cb6..eed1869744a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -4,14 +4,46 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest", namespace = JsPackage.GLOBAL) public class RemoteFileSourceMetaRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static RemoteFileSourceMetaRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static RemoteFileSourceMetaRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + public static native RemoteFileSourceMetaRequest deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( @@ -20,11 +52,16 @@ public static native RemoteFileSourceMetaRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceMetaRequest message, Object writer); - public native void clearResourceName(); + public static native RemoteFileSourceMetaRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceMetaRequest msg); public native String getResourceName(); public native Uint8Array serializeBinary(); public native void setResourceName(String value); + + public native RemoteFileSourceMetaRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceMetaRequest.ToObjectReturnType0 toObject(boolean includeInstance); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java index 157c0fad3fb..d06ac259b4a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java @@ -4,14 +4,210 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse", namespace = JsPackage.GLOBAL) public class RemoteFileSourceMetaResponse { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.GetContentUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetContentValueUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.SetContentValueUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType.GetContentUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceMetaResponse.ToObjectReturnType.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent(RemoteFileSourceMetaResponse.ToObjectReturnType.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface GetContentUnionType { + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType0.GetContentUnionType of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default String asString() { + return Js.asString(this); + } + + @JsOverlay + default Uint8Array asUint8Array() { + return Js.cast(this); + } + + @JsOverlay + default boolean isString() { + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return (Object) this instanceof Uint8Array; + } + } + + @JsOverlay + static RemoteFileSourceMetaResponse.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceMetaResponse.ToObjectReturnType0.GetContentUnionType getContent(); + + @JsProperty + String getError(); + + @JsProperty + boolean isFound(); + + @JsProperty + void setContent(RemoteFileSourceMetaResponse.ToObjectReturnType0.GetContentUnionType content); + + @JsOverlay + default void setContent(String content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsOverlay + default void setContent(Uint8Array content) { + setContent( + Js.uncheckedCast( + content)); + } + + @JsProperty + void setError(String error); + + @JsProperty + void setFound(boolean found); + } + public static native RemoteFileSourceMetaResponse deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceMetaResponse deserializeBinaryFromReader( @@ -20,24 +216,38 @@ public static native RemoteFileSourceMetaResponse deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceMetaResponse message, Object writer); - public native void clearContent(); - - public native void clearFound(); + public static native RemoteFileSourceMetaResponse.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceMetaResponse msg); - public native void clearError(); + public native RemoteFileSourceMetaResponse.GetContentUnionType getContent(); - public native Uint8Array getContent(); + public native String getContent_asB64(); - public native boolean getFound(); + public native Uint8Array getContent_asU8(); public native String getError(); + public native boolean getFound(); + public native Uint8Array serializeBinary(); - public native void setContent(Uint8Array value); + public native void setContent(RemoteFileSourceMetaResponse.SetContentValueUnionType value); - public native void setFound(boolean value); + @JsOverlay + public final void setContent(String value) { + setContent(Js.uncheckedCast(value)); + } + + @JsOverlay + public final void setContent(Uint8Array value) { + setContent(Js.uncheckedCast(value)); + } public native void setError(String value); -} + public native void setFound(boolean value); + + public native RemoteFileSourceMetaResponse.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceMetaResponse.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index bd48a2068e5..b87d9dcb742 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -82,18 +82,18 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType create() { return Js.uncheckedCast(JsPropertyMap.of()); } + @JsProperty + String getPluginName(); + @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType getResultId(); @JsProperty - String getPluginName(); + void setPluginName(String pluginName); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType.ResultIdFieldType resultId); - - @JsProperty - void setPluginName(String pluginName); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -161,18 +161,18 @@ static RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 create() { return Js.uncheckedCast(JsPropertyMap.of()); } + @JsProperty + String getPluginName(); + @JsProperty RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType getResultId(); @JsProperty - String getPluginName(); + void setPluginName(String pluginName); @JsProperty void setResultId( RemoteFileSourcePluginFetchRequest.ToObjectReturnType0.ResultIdFieldType resultId); - - @JsProperty - void setPluginName(String pluginName); } public static native RemoteFileSourcePluginFetchRequest deserializeBinary(Uint8Array bytes); @@ -188,20 +188,20 @@ public static native RemoteFileSourcePluginFetchRequest.ToObjectReturnType toObj public native void clearResultId(); - public native Ticket getResultId(); - public native String getPluginName(); + public native Ticket getResultId(); + public native boolean hasResultId(); public native Uint8Array serializeBinary(); + public native void setPluginName(String value); + public native void setResultId(); public native void setResultId(Ticket value); - public native void setPluginName(String value); - public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourcePluginFetchRequest.ToObjectReturnType0 toObject( diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index 4ce84895ce4..f659e2336b2 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -4,14 +4,130 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest", namespace = JsPackage.GLOBAL) public class RemoteFileSourceServerRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaRequestFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextResponseFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType getMetaRequest(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); + + @JsProperty + void setMetaRequest( + RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType metaRequest); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContextResponse( + RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType setExecutionContextResponse); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface MetaRequestFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + String getResourceName(); + + @JsProperty + void setResourceName(String resourceName); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface SetExecutionContextResponseFieldType { + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + + @JsOverlay + static RemoteFileSourceServerRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType getMetaRequest(); + + @JsProperty + String getRequestId(); + + @JsProperty + RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); + + @JsProperty + void setMetaRequest( + RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType metaRequest); + + @JsProperty + void setRequestId(String requestId); + + @JsProperty + void setSetExecutionContextResponse( + RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType setExecutionContextResponse); + } + public static native RemoteFileSourceServerRequest deserializeBinary(Uint8Array bytes); public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( @@ -20,16 +136,19 @@ public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( RemoteFileSourceServerRequest message, Object writer); - public native void clearRequestId(); + public static native RemoteFileSourceServerRequest.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceServerRequest msg); public native void clearMetaRequest(); public native void clearSetExecutionContextResponse(); - public native String getRequestId(); - public native RemoteFileSourceMetaRequest getMetaRequest(); + public native int getRequestCase(); + + public native String getRequestId(); + public native SetExecutionContextResponse getSetExecutionContextResponse(); public native boolean hasMetaRequest(); @@ -38,14 +157,17 @@ public static native void serializeBinaryToWriter( public native Uint8Array serializeBinary(); - public native void setRequestId(String value); - public native void setMetaRequest(); public native void setMetaRequest(RemoteFileSourceMetaRequest value); + public native void setRequestId(String value); + public native void setSetExecutionContextResponse(); public native void setSetExecutionContextResponse(SetExecutionContextResponse value); -} + public native RemoteFileSourceServerRequest.ToObjectReturnType0 toObject(); + + public native RemoteFileSourceServerRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 8967670152f..e8c069eb1ee 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -5,14 +5,56 @@ import elemental2.core.JsArray; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest", namespace = JsPackage.GLOBAL) public class SetExecutionContextRequest { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static SetExecutionContextRequest.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static SetExecutionContextRequest.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + JsArray getResourcePathsList(); + + @JsProperty + void setResourcePathsList(JsArray resourcePathsList); + + @JsOverlay + default void setResourcePathsList(String[] resourcePathsList) { + setResourcePathsList(Js.>uncheckedCast(resourcePathsList)); + } + } + public static native SetExecutionContextRequest deserializeBinary(Uint8Array bytes); public static native SetExecutionContextRequest deserializeBinaryFromReader( @@ -21,6 +63,13 @@ public static native SetExecutionContextRequest deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextRequest message, Object writer); + public static native SetExecutionContextRequest.ToObjectReturnType toObject( + boolean includeInstance, SetExecutionContextRequest msg); + + public native String addResourcePaths(String value, double index); + + public native String addResourcePaths(String value); + public native void clearResourcePathsList(); public native JsArray getResourcePathsList(); @@ -29,8 +78,12 @@ public static native void serializeBinaryToWriter( public native void setResourcePathsList(JsArray value); - public native String addResourcePaths(String value); + @JsOverlay + public final void setResourcePathsList(String[] value) { + setResourcePathsList(Js.>uncheckedCast(value)); + } - public native String addResourcePaths(String value, double index); -} + public native SetExecutionContextRequest.ToObjectReturnType0 toObject(); + public native SetExecutionContextRequest.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java index 8c27b48e69a..09d9d1861d8 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -4,14 +4,46 @@ package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsProperty; import jsinterop.annotations.JsType; +import jsinterop.base.Js; +import jsinterop.base.JsPropertyMap; @JsType( isNative = true, name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse", namespace = JsPackage.GLOBAL) public class SetExecutionContextResponse { + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType { + @JsOverlay + static SetExecutionContextResponse.ToObjectReturnType create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + + @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) + public interface ToObjectReturnType0 { + @JsOverlay + static SetExecutionContextResponse.ToObjectReturnType0 create() { + return Js.uncheckedCast(JsPropertyMap.of()); + } + + @JsProperty + boolean isSuccess(); + + @JsProperty + void setSuccess(boolean success); + } + public static native SetExecutionContextResponse deserializeBinary(Uint8Array bytes); public static native SetExecutionContextResponse deserializeBinaryFromReader( @@ -20,13 +52,16 @@ public static native SetExecutionContextResponse deserializeBinaryFromReader( public static native void serializeBinaryToWriter( SetExecutionContextResponse message, Object writer); - public native void clearSuccess(); + public static native SetExecutionContextResponse.ToObjectReturnType toObject( + boolean includeInstance, SetExecutionContextResponse msg); public native boolean getSuccess(); public native Uint8Array serializeBinary(); - public native void setSuccess(boolean value); -} + public native SetExecutionContextResponse.ToObjectReturnType0 toObject(); + + public native SetExecutionContextResponse.ToObjectReturnType0 toObject(boolean includeInstance); +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java new file mode 100644 index 00000000000..46db67bea6b --- /dev/null +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java @@ -0,0 +1,15 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceclientrequest; + +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; + +@JsType( + isNative = true, + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest.RequestCase", + namespace = JsPackage.GLOBAL) +public class RequestCase { + public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT, TEST_COMMAND; +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java similarity index 65% rename from web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java rename to web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java index c263a6194cb..72cad9f6ff9 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourcepluginrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java @@ -1,17 +1,15 @@ // // Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending // -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourcepluginrequest; +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceserverrequest; import jsinterop.annotations.JsPackage; import jsinterop.annotations.JsType; @JsType( isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginRequest.RequestCase", + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest.RequestCase", namespace = JsPackage.GLOBAL) public class RequestCase { - public static int META, - REQUEST_NOT_SET, - SET_CONNECTION_ID; + public static int META_REQUEST, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT_RESPONSE; } From aa2e5e9b0fee451f8b1993ab18de3fb401454730 Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Fri, 19 Dec 2025 14:00:57 -0600 Subject: [PATCH 30/78] FIxing compile error --- .../api/remotefilesource/JsRemoteFileSourceService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index c91ce77fefa..845abafb560 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -283,7 +283,7 @@ public String getResourceName() { /** * Responds to this resource request with the given content. * - * @param content the resource content (string, ArrayBuffer, or typed array), or null if not found + * @param content the resource content (string or Uint8Array), or null if not found */ @JsMethod public void respond(@JsNullable Object content) { @@ -299,11 +299,11 @@ public void respond(@JsNullable Object content) { // Convert content to bytes if (content instanceof String) { - response.setContent(stringToUtf8((String) content)); + response.setContent((String) content); } else if (content instanceof Uint8Array) { response.setContent((Uint8Array) content); } else { - response.setContent(Js.uncheckedCast(content)); + throw new IllegalArgumentException("Content must be a String, Uint8Array, or null"); } } From 1e4b4acc7471f3ee5c3b7d28959c83af4a4299ec Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 23 Dec 2025 17:06:26 -0600 Subject: [PATCH 31/78] Replaced util with TextEncoder.encode (#DH-20578) --- .../JsRemoteFileSourceService.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 845abafb560..c1c953e876d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -8,6 +8,7 @@ import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.Uint8Array; import elemental2.dom.DomGlobal; +import elemental2.dom.TextEncoder; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; @@ -299,7 +300,8 @@ public void respond(@JsNullable Object content) { // Convert content to bytes if (content instanceof String) { - response.setContent((String) content); + TextEncoder textEncoder = new TextEncoder(); + response.setContent(textEncoder.encode((String) content)); } else if (content instanceof Uint8Array) { response.setContent((Uint8Array) content); } else { @@ -325,7 +327,8 @@ public void respond(@JsNullable Object content) { */ private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { // Encode the type_url string to UTF-8 bytes - Uint8Array typeUrlBytes = stringToUtf8(typeUrl); + TextEncoder textEncoder = new TextEncoder(); + Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); // Calculate sizes for protobuf encoding // Field 1 (type_url): tag + length + data @@ -357,15 +360,6 @@ private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { return result; } - private static Uint8Array stringToUtf8(String str) { - // Simple UTF-8 encoding for ASCII-compatible strings - Uint8Array bytes = new Uint8Array(str.length()); - for (int i = 0; i < str.length(); i++) { - bytes.setAt(i, (double) str.charAt(i)); - } - return bytes; - } - private static int sizeOfVarint(int value) { if (value < 0) return 10; From a20d1514f4b27e39fc1bee0f3e564350d411693e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 09:48:39 -0600 Subject: [PATCH 32/78] Cleanup (#DH-20578) --- .../JsRemoteFileSourceService.java | 162 ++++++++++++++---- 1 file changed, 129 insertions(+), 33 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index c1c953e876d..b2dd397a274 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -318,67 +318,163 @@ public void respond(@JsNullable Object content) { } } + /** + * Calculates the total size needed for a protobuf length-delimited field. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param tag the protobuf field tag (field number << 3 | wire type) + * @param dataLength the length of the data in bytes + * @return the total number of bytes needed for this field + */ + private static int calculateFieldSize(int tag, int dataLength) { + return sizeOfVarint(tag) + sizeOfVarint(dataLength) + dataLength; + } + + /** + * Calculates how many bytes a varint encoding will require for the given value. + *

+ * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is + * a continuation flag). This means: + *

    + *
  • 1 byte: 0 to 127 (2^7 - 1)
    • + *
  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • + *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • + *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • + *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • + *
  • 10 bytes: negative numbers (due to sign extension)
    • + *
+ * + * @param value the integer value to encode + * @return the number of bytes required to encode the value as a varint + */ + private static int sizeOfVarint(int value) { + if (value < 0) + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < 128) // 2^7 + return 1; + if (value < 16384) // 2^14 + return 2; + if (value < 2097152) // 2^21 + return 3; + if (value < 268435456) // 2^28 + return 4; + return 5; // 2^35 (max for positive 32-bit int) + } + /** * Wraps a protobuf message in a google.protobuf.Any message. + *

+ * The google.protobuf.Any message has two fields: + *

    + *
  • Field 1: type_url (string) - identifies the type of message contained
    • + *
  • Field 2: value (bytes) - the actual serialized message
    • + *
+ *

+ * This method manually encodes the Any message in protobuf binary format since the client-side + * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. * * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") * @param messageBytes the serialized protobuf message bytes * @return the serialized Any message containing the wrapped message */ private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { + // Protobuf tag constants for google.protobuf.Any message fields + // Tag format: (field_number << 3) | wire_type + // wire_type=2 means length-delimited (for strings/bytes) + final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 + final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 + // Encode the type_url string to UTF-8 bytes TextEncoder textEncoder = new TextEncoder(); Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); - // Calculate sizes for protobuf encoding - // Field 1 (type_url): tag + length + data - int typeUrlTag = (1 << 3) | 2; // field 1, wire type 2 (length-delimited) - int typeUrlFieldSize = sizeOfVarint(typeUrlTag) + sizeOfVarint(typeUrlBytes.length) + typeUrlBytes.length; - - // Field 2 (value): tag + length + data - int valueTag = (2 << 3) | 2; // field 2, wire type 2 (length-delimited) - int valueFieldSize = sizeOfVarint(valueTag) + sizeOfVarint(messageBytes.length) + messageBytes.length; + // Calculate sizes for protobuf binary encoding + int typeUrlFieldSize = calculateFieldSize(TYPE_URL_TAG, typeUrlBytes.length); + int valueFieldSize = calculateFieldSize(VALUE_TAG, messageBytes.length); + // Allocate buffer for the complete Any message int totalSize = typeUrlFieldSize + valueFieldSize; Uint8Array result = new Uint8Array(totalSize); int pos = 0; - // Write type_url field - pos = writeVarint(result, pos, typeUrlTag); - pos = writeVarint(result, pos, typeUrlBytes.length); - for (int i = 0; i < typeUrlBytes.length; i++) { - result.setAt(pos++, typeUrlBytes.getAt(i)); - } + // Write field 1 (type_url) in protobuf binary format + pos = writeField(result, pos, TYPE_URL_TAG, typeUrlBytes); - // Write value field - pos = writeVarint(result, pos, valueTag); - pos = writeVarint(result, pos, messageBytes.length); - for (int i = 0; i < messageBytes.length; i++) { - result.setAt(pos++, messageBytes.getAt(i)); - } + // Write field 2 (value) in protobuf binary format + writeField(result, pos, VALUE_TAG, messageBytes); return result; } - private static int sizeOfVarint(int value) { - if (value < 0) - return 10; - if (value < 128) - return 1; - if (value < 16384) - return 2; - if (value < 2097152) - return 3; - if (value < 268435456) - return 4; - return 5; + /** + * Writes a complete protobuf length-delimited field to the buffer. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param tag the protobuf field tag + * @param data the data bytes to write + * @return the new position after writing the complete field + */ + private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array data) { + // Write tag and length + pos = writeVarint(buffer, pos, tag); + pos = writeVarint(buffer, pos, data.length); + // Write data bytes + for (int i = 0; i < data.length; i++) { + buffer.setAt(pos++, data.getAt(i)); + } + return pos; } + + /** + * Writes a value to the buffer as a protobuf varint (variable-length integer). + *

+ * Varint encoding works by: + *

    + *
  1. Taking the lowest 7 bits of the value
    2. + *
  2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
    3. + *
  3. Writing the byte to the buffer
    4. + *
  4. Shifting the value right by 7 bits
    5. + *
  5. Repeating until the value is less than 128
    6. + *
  6. Writing the final byte without the continuation flag (8th bit = 0)
    7. + *
+ *

+ * Example: encoding 300 + *

    + *
  • 300 in binary: 100101100
    • + *
  • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
    • + *
  • Shift: 300 >>> 7 = 2
    • + *
  • Second byte: 2 (no continuation flag)
    • + *
  • Result: [172, 2]
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param value the value to encode + * @return the new position after writing + */ private static int writeVarint(Uint8Array buffer, int pos, int value) { while (value >= 128) { + // Extract lowest 7 bits and set continuation flag (8th bit = 1) buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); - value >>>= 7; + // Shift right by 7 to process next chunk + value >>>= 7; // Unsigned right shift to handle large positive values } + // Write final byte (no continuation flag, 8th bit = 0) buffer.setAt(pos++, (double) value); return pos; } From 37f427a59acf92319a049db1ee6e2e4ffdfa0f89 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 10:17:41 -0600 Subject: [PATCH 33/78] Split out JsProtobufUtils (#DH-20578) --- .../web/client/api/JsProtobufUtils.java | 178 ++++++++++++++++++ .../JsRemoteFileSourceService.java | 164 +--------------- 2 files changed, 180 insertions(+), 162 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java new file mode 100644 index 00000000000..7413aedd954 --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java @@ -0,0 +1,178 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api; + +import elemental2.core.Uint8Array; +import elemental2.dom.TextEncoder; + +/** + * Utility methods for working with protobuf messages in the client-side JavaScript environment. + */ +public class JsProtobufUtils { + + private JsProtobufUtils() { + // Utility class, no instantiation + } + + /** + * Wraps a protobuf message in a google.protobuf.Any message. + *

+ * The google.protobuf.Any message has two fields: + *

    + *
  • Field 1: type_url (string) - identifies the type of message contained
    • + *
  • Field 2: value (bytes) - the actual serialized message
    • + *
+ *

+ * This method manually encodes the Any message in protobuf binary format since the client-side + * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. + * + * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") + * @param messageBytes the serialized protobuf message bytes + * @return the serialized Any message containing the wrapped message + */ + public static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { + // Protobuf tag constants for google.protobuf.Any message fields + // Tag format: (field_number << 3) | wire_type + // wire_type=2 means length-delimited (for strings/bytes) + final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 + final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 + + // Encode the type_url string to UTF-8 bytes + TextEncoder textEncoder = new TextEncoder(); + Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); + + // Calculate sizes for protobuf binary encoding + int typeUrlFieldSize = calculateFieldSize(TYPE_URL_TAG, typeUrlBytes.length); + int valueFieldSize = calculateFieldSize(VALUE_TAG, messageBytes.length); + + // Allocate buffer for the complete Any message + int totalSize = typeUrlFieldSize + valueFieldSize; + Uint8Array result = new Uint8Array(totalSize); + int pos = 0; + + // Write field 1 (type_url) in protobuf binary format + pos = writeField(result, pos, TYPE_URL_TAG, typeUrlBytes); + + // Write field 2 (value) in protobuf binary format + writeField(result, pos, VALUE_TAG, messageBytes); + + return result; + } + + /** + * Calculates the total size needed for a protobuf length-delimited field. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param tag the protobuf field tag (field number << 3 | wire type) + * @param dataLength the length of the data in bytes + * @return the total number of bytes needed for this field + */ + private static int calculateFieldSize(int tag, int dataLength) { + return sizeOfVarint(tag) + sizeOfVarint(dataLength) + dataLength; + } + + /** + * Calculates how many bytes a varint encoding will require for the given value. + *

+ * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is + * a continuation flag). This means: + *

    + *
  • 1 byte: 0 to 127 (2^7 - 1)
    • + *
  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • + *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • + *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • + *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • + *
  • 10 bytes: negative numbers (due to sign extension)
    • + *
+ * + * @param value the integer value to encode + * @return the number of bytes required to encode the value as a varint + */ + private static int sizeOfVarint(int value) { + if (value < 0) + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < 128) // 2^7 + return 1; + if (value < 16384) // 2^14 + return 2; + if (value < 2097152) // 2^21 + return 3; + if (value < 268435456) // 2^28 + return 4; + return 5; // 2^35 (max for positive 32-bit int) + } + + /** + * Writes a complete protobuf length-delimited field to the buffer. + *

+ * A length-delimited field consists of: + *

    + *
  • Tag (field number + wire type) encoded as a varint
    • + *
  • Length of the data encoded as a varint
    • + *
  • The actual data bytes
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param tag the protobuf field tag + * @param data the data bytes to write + * @return the new position after writing the complete field + */ + private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array data) { + // Write tag and length + pos = writeVarint(buffer, pos, tag); + pos = writeVarint(buffer, pos, data.length); + // Write data bytes + for (int i = 0; i < data.length; i++) { + buffer.setAt(pos++, data.getAt(i)); + } + return pos; + } + + /** + * Writes a value to the buffer as a protobuf varint (variable-length integer). + *

+ * Varint encoding works by: + *

    + *
  1. Taking the lowest 7 bits of the value
    2. + *
  2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
    3. + *
  3. Writing the byte to the buffer
    4. + *
  4. Shifting the value right by 7 bits
    5. + *
  5. Repeating until the value is less than 128
    6. + *
  6. Writing the final byte without the continuation flag (8th bit = 0)
    7. + *
+ *

+ * Example: encoding 300 + *

    + *
  • 300 in binary: 100101100
    • + *
  • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
    • + *
  • Shift: 300 >>> 7 = 2
    • + *
  • Second byte: 2 (no continuation flag)
    • + *
  • Result: [172, 2]
    • + *
+ * + * @param buffer the buffer to write to + * @param pos the starting position in the buffer + * @param value the value to encode + * @return the new position after writing + */ + private static int writeVarint(Uint8Array buffer, int pos, int value) { + while (value >= 128) { + // Extract lowest 7 bits and set continuation flag (8th bit = 1) + buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); + // Shift right by 7 to process next chunk + value >>>= 7; // Unsigned right shift to handle large positive values + } + // Write final byte (no continuation flag, 8th bit = 0) + buffer.setAt(pos++, (double) value); + return pos; + } +} + diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index b2dd397a274..dffeefae01d 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -22,6 +22,7 @@ import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.TypedTicket; import io.deephaven.web.client.api.Callbacks; +import io.deephaven.web.client.api.JsProtobufUtils; import io.deephaven.web.client.api.event.Event; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.event.HasEventHandling; @@ -81,7 +82,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); // Wrap in google.protobuf.Any with the proper typeUrl - Uint8Array anyWrappedBytes = wrapInAny( + Uint8Array anyWrappedBytes = JsProtobufUtils.wrapInAny( "type.googleapis.com/io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest", innerRequestBytes); @@ -317,165 +318,4 @@ public void respond(@JsNullable Object content) { sendClientRequest(clientRequest); } } - - /** - * Calculates the total size needed for a protobuf length-delimited field. - *

- * A length-delimited field consists of: - *

    - *
  • Tag (field number + wire type) encoded as a varint
    • - *
  • Length of the data encoded as a varint
    • - *
  • The actual data bytes
    • - *
- * - * @param tag the protobuf field tag (field number << 3 | wire type) - * @param dataLength the length of the data in bytes - * @return the total number of bytes needed for this field - */ - private static int calculateFieldSize(int tag, int dataLength) { - return sizeOfVarint(tag) + sizeOfVarint(dataLength) + dataLength; - } - - /** - * Calculates how many bytes a varint encoding will require for the given value. - *

- * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is - * a continuation flag). This means: - *

    - *
  • 1 byte: 0 to 127 (2^7 - 1)
    • - *
  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • - *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • - *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • - *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • - *
  • 10 bytes: negative numbers (due to sign extension)
    • - *
- * - * @param value the integer value to encode - * @return the number of bytes required to encode the value as a varint - */ - private static int sizeOfVarint(int value) { - if (value < 0) - return 10; // Negative numbers use sign extension, always 10 bytes - if (value < 128) // 2^7 - return 1; - if (value < 16384) // 2^14 - return 2; - if (value < 2097152) // 2^21 - return 3; - if (value < 268435456) // 2^28 - return 4; - return 5; // 2^35 (max for positive 32-bit int) - } - - /** - * Wraps a protobuf message in a google.protobuf.Any message. - *

- * The google.protobuf.Any message has two fields: - *

    - *
  • Field 1: type_url (string) - identifies the type of message contained
    • - *
  • Field 2: value (bytes) - the actual serialized message
    • - *
- *

- * This method manually encodes the Any message in protobuf binary format since the client-side - * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. - * - * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") - * @param messageBytes the serialized protobuf message bytes - * @return the serialized Any message containing the wrapped message - */ - private static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { - // Protobuf tag constants for google.protobuf.Any message fields - // Tag format: (field_number << 3) | wire_type - // wire_type=2 means length-delimited (for strings/bytes) - final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 - final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 - - // Encode the type_url string to UTF-8 bytes - TextEncoder textEncoder = new TextEncoder(); - Uint8Array typeUrlBytes = textEncoder.encode(typeUrl); - - // Calculate sizes for protobuf binary encoding - int typeUrlFieldSize = calculateFieldSize(TYPE_URL_TAG, typeUrlBytes.length); - int valueFieldSize = calculateFieldSize(VALUE_TAG, messageBytes.length); - - // Allocate buffer for the complete Any message - int totalSize = typeUrlFieldSize + valueFieldSize; - Uint8Array result = new Uint8Array(totalSize); - int pos = 0; - - // Write field 1 (type_url) in protobuf binary format - pos = writeField(result, pos, TYPE_URL_TAG, typeUrlBytes); - - // Write field 2 (value) in protobuf binary format - writeField(result, pos, VALUE_TAG, messageBytes); - - return result; - } - - /** - * Writes a complete protobuf length-delimited field to the buffer. - *

- * A length-delimited field consists of: - *

    - *
  • Tag (field number + wire type) encoded as a varint
    • - *
  • Length of the data encoded as a varint
    • - *
  • The actual data bytes
    • - *
- * - * @param buffer the buffer to write to - * @param pos the starting position in the buffer - * @param tag the protobuf field tag - * @param data the data bytes to write - * @return the new position after writing the complete field - */ - private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array data) { - // Write tag and length - pos = writeVarint(buffer, pos, tag); - pos = writeVarint(buffer, pos, data.length); - // Write data bytes - for (int i = 0; i < data.length; i++) { - buffer.setAt(pos++, data.getAt(i)); - } - return pos; - } - - - /** - * Writes a value to the buffer as a protobuf varint (variable-length integer). - *

- * Varint encoding works by: - *

    - *
  1. Taking the lowest 7 bits of the value
    2. - *
  2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
    3. - *
  3. Writing the byte to the buffer
    4. - *
  4. Shifting the value right by 7 bits
    5. - *
  5. Repeating until the value is less than 128
    6. - *
  6. Writing the final byte without the continuation flag (8th bit = 0)
    7. - *
- *

- * Example: encoding 300 - *

    - *
  • 300 in binary: 100101100
    • - *
  • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
    • - *
  • Shift: 300 >>> 7 = 2
    • - *
  • Second byte: 2 (no continuation flag)
    • - *
  • Result: [172, 2]
    • - *
- * - * @param buffer the buffer to write to - * @param pos the starting position in the buffer - * @param value the value to encode - * @return the new position after writing - */ - private static int writeVarint(Uint8Array buffer, int pos, int value) { - while (value >= 128) { - // Extract lowest 7 bits and set continuation flag (8th bit = 1) - buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); - // Shift right by 7 to process next chunk - value >>>= 7; // Unsigned right shift to handle large positive values - } - // Write final byte (no continuation flag, 8th bit = 0) - buffer.setAt(pos++, (double) value); - return pos; - } } From aba50b1a9b53b07c728cc56fb98e642bd879482e Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 10:49:04 -0600 Subject: [PATCH 34/78] Made canSourceResource synchronous (#DH-20578) --- .../engine/util/RemoteFileSourceClassLoader.java | 15 ++++++--------- .../engine/util/RemoteFileSourceProvider.java | 4 ++-- .../RemoteFileSourceMessageStream.java | 14 +++++++------- .../JsRemoteFileSourceService.java | 14 -------------- 4 files changed, 15 insertions(+), 32 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index a3235ad9450..efc658e5a1e 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -24,7 +24,8 @@ * */ public class RemoteFileSourceClassLoader extends ClassLoader { - private static final boolean DEBUG = Boolean.getBoolean("RemoteFileSourceClassLoader.debug"); + private static final long RESOURCE_TIMEOUT_SECONDS = 5; + private static volatile RemoteFileSourceClassLoader instance; private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); @@ -52,15 +53,11 @@ protected URL findResource(String name) { continue; } try { - Boolean canSource = provider.canSourceResource(name) - .orTimeout(5, TimeUnit.SECONDS) - .get(); - - if (Boolean.TRUE.equals(canSource)) { + if (provider.canSourceResource(name)) { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); } - } catch (Exception e) { - // Continue to next provider + } catch (java.net.MalformedURLException e) { + // Continue to next provider if URL creation fails } } return super.findResource(name); @@ -103,7 +100,7 @@ public void connect() throws IOException { if (!connected) { try { content = provider.requestResource(resourceName) - .orTimeout(5, TimeUnit.SECONDS) + .orTimeout(RESOURCE_TIMEOUT_SECONDS, TimeUnit.SECONDS) .get(); connected = true; } catch (Exception e) { diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index 4c5408d7833..02e5eee5feb 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -15,9 +15,9 @@ public interface RemoteFileSourceProvider { * Check if this provider can source the given resource. * * @param resourceName the name of the resource to check (e.g., "com/example/MyClass.groovy") - * @return a CompletableFuture that resolves to true if this provider can handle the resource, false otherwise + * @return true if this provider can handle the resource, false otherwise */ - CompletableFuture canSourceResource(String resourceName); + boolean canSourceResource(String resourceName); /** * Request a resource from the remote source. diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 1067966d2c0..4161d4a937d 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -51,36 +51,36 @@ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) // RemoteFileSourceProvider interface implementation - each instance is a provider @Override - public java.util.concurrent.CompletableFuture canSourceResource(String resourceName) { + public boolean canSourceResource(String resourceName) { // Only active if this instance is the currently active message stream if (!isActive()) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } // Only handle .groovy source files, not compiled .class files if (!resourceName.endsWith(".groovy")) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } RemoteFileSourceExecutionContext context = executionContext; if (context == null || context.getActiveMessageStream() != this) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } java.util.List resourcePaths = context.getResourcePaths(); if (resourcePaths.isEmpty()) { - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } // Resource names from ClassLoader always use forward slashes, not backslashes for (String contextResourcePath : resourcePaths) { if (resourceName.equals(contextResourcePath)) { log.info().append("✅ Can source: ").append(resourceName).endl(); - return java.util.concurrent.CompletableFuture.completedFuture(true); + return true; } } - return java.util.concurrent.CompletableFuture.completedFuture(false); + return false; } @Override diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index dffeefae01d..dd0d6ef0f09 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -179,20 +179,6 @@ private Promise connect() { return widget.refetch().then(w -> Promise.resolve(this)); } - /** - * Test method to verify bidirectional communication. - * Sends a test command to the server, which will request a resource back from the client. - * - * @param resourceName the resource name to use for the test (e.g., "com/example/Test.java") - */ - @JsMethod - public void testBidirectionalCommunication(String resourceName) { - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); - clientRequest.setRequestId(""); // Empty request_id for test commands - clientRequest.setTestCommand("TEST:" + resourceName); - sendClientRequest(clientRequest); - } - /** * Sets the execution context on the server to identify this message stream as active * for script execution. From 576850c0682a301d5fc720d5f3a2f0de215515c4 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 24 Dec 2025 10:54:10 -0600 Subject: [PATCH 35/78] renamed arg (#DH-20578) --- .../io/deephaven/engine/util/RemoteFileSourceClassLoader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index efc658e5a1e..25c82b41147 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -76,8 +76,8 @@ private static class RemoteFileURLStreamHandler extends URLStreamHandler { } @Override - protected URLConnection openConnection(URL u) { - return new RemoteFileURLConnection(u, provider, resourceName); + protected URLConnection openConnection(URL url) { + return new RemoteFileURLConnection(url, provider, resourceName); } } From ef00be5a93db6b7591d3443ee521cc2686444eea Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 10:22:05 -0600 Subject: [PATCH 36/78] Cleanup (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 73 ++++++++++++++++++- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 25c82b41147..3add4e468cc 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -29,37 +29,67 @@ public class RemoteFileSourceClassLoader extends ClassLoader { private static volatile RemoteFileSourceClassLoader instance; private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); + /** + * Constructs a new RemoteFileSourceClassLoader with the specified parent class loader. + * + * @param parent the parent class loader for delegation + */ public RemoteFileSourceClassLoader(ClassLoader parent) { super(parent); instance = this; } + /** + * Returns the singleton instance of the RemoteFileSourceClassLoader. + * + * @return the singleton instance, or null if not yet initialized + */ public static RemoteFileSourceClassLoader getInstance() { return instance; } + /** + * Registers a new provider that can source remote resources. + * + * @param provider the provider to register + */ public void registerProvider(RemoteFileSourceProvider provider) { providers.add(provider); } + /** + * Unregisters a previously registered provider. + * + * @param provider the provider to unregister + */ public void unregisterProvider(RemoteFileSourceProvider provider) { providers.remove(provider); } + /** + * Finds the resource with the specified name by checking registered providers. + * + *

This method iterates through all registered providers to see if any can source the requested resource. + * If a provider can handle the resource, a custom URL with protocol "remotefile://" is returned. + * If no provider can handle the resource, the request is delegated to the parent class loader. + * + * @param name the resource name + * @return a URL for reading the resource, or null if the resource could not be found + */ @Override protected URL findResource(String name) { for (RemoteFileSourceProvider provider : providers) { - if (!provider.isActive()) { + if (!provider.isActive() || !provider.canSourceResource(name)) { continue; } + try { - if (provider.canSourceResource(name)) { - return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); - } + return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); } catch (java.net.MalformedURLException e) { // Continue to next provider if URL creation fails } } + return super.findResource(name); } @@ -70,11 +100,23 @@ private static class RemoteFileURLStreamHandler extends URLStreamHandler { private final RemoteFileSourceProvider provider; private final String resourceName; + /** + * Constructs a new RemoteFileURLStreamHandler for the specified provider and resource. + * + * @param provider the provider that will source the resource + * @param resourceName the name of the resource to fetch + */ RemoteFileURLStreamHandler(RemoteFileSourceProvider provider, String resourceName) { this.provider = provider; this.resourceName = resourceName; } + /** + * Opens a connection to the resource referenced by this URL. + * + * @param url the URL to open a connection to + * @return a URLConnection to the specified URL + */ @Override protected URLConnection openConnection(URL url) { return new RemoteFileURLConnection(url, provider, resourceName); @@ -89,12 +131,27 @@ private static class RemoteFileURLConnection extends URLConnection { private final String resourceName; private byte[] content; + /** + * Constructs a new RemoteFileURLConnection for the specified URL, provider, and resource. + * + * @param url the URL to connect to + * @param provider the provider that will source the resource + * @param resourceName the name of the resource to fetch + */ RemoteFileURLConnection(URL url, RemoteFileSourceProvider provider, String resourceName) { super(url); this.provider = provider; this.resourceName = resourceName; } + /** + * Opens a connection to the resource by requesting it from the provider. + * + *

This method fetches the resource bytes from the provider with a timeout of + * {@value #RESOURCE_TIMEOUT_SECONDS} seconds. If already connected, this method does nothing. + * + * @throws IOException if the connection fails or times out + */ @Override public void connect() throws IOException { if (!connected) { @@ -109,6 +166,14 @@ public void connect() throws IOException { } } + /** + * Returns an input stream that reads from this connection's resource. + * + *

This method ensures the connection is established before returning the stream. + * + * @return an input stream that reads from the resource + * @throws IOException if the connection cannot be established or if the resource has no content + */ @Override public InputStream getInputStream() throws IOException { connect(); From 10c8f8e24d4ec0fde1ca317d8739e26cc85776a5 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 10:26:18 -0600 Subject: [PATCH 37/78] Sorted members (#DH-20578) --- .../engine/util/RemoteFileSourceProvider.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index 02e5eee5feb..377172ea605 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -20,18 +20,18 @@ public interface RemoteFileSourceProvider { boolean canSourceResource(String resourceName); /** - * Request a resource from the remote source. + * Check if this provider is currently active and should be used for resource requests. * - * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy") - * @return a CompletableFuture containing the resource bytes, or null if not found + * @return true if this provider is active, false otherwise */ - CompletableFuture requestResource(String resourceName); + boolean isActive(); /** - * Check if this provider is currently active and should be used for resource requests. + * Request a resource from the remote source. * - * @return true if this provider is active, false otherwise + * @param resourceName the name of the resource to fetch (e.g., "com/example/MyClass.groovy") + * @return a CompletableFuture containing the resource bytes, or null if not found */ - boolean isActive(); + CompletableFuture requestResource(String resourceName); } From 2eeba31f6719c742baf365f965fb5a477c5c1d66 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 11:49:46 -0600 Subject: [PATCH 38/78] Cleanup RemoteFileSourceCommandResolver (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 101 ++++++++++++------ 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index c1d973609af..149edf63b13 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -20,7 +20,6 @@ import io.deephaven.server.session.TicketRouter; import io.deephaven.server.session.WantsTicketRouter; -import io.grpc.Status; import io.grpc.StatusRuntimeException; import org.apache.arrow.flight.impl.Flight; import org.jetbrains.annotations.Nullable; @@ -62,7 +61,7 @@ private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any co * @param data the ByteString data to parse * @return the parsed Any message, or null if parsing fails */ - private static Any parseOrNull(final ByteString data) { + private static Any parseAsAnyOrNull(final ByteString data) { try { return Any.parseFrom(data); } catch (final InvalidProtocolBufferException e) { @@ -71,11 +70,11 @@ private static Any parseOrNull(final ByteString data) { } /** - * Exports a PluginMarker singleton based on the fetch request. + * Exports the PluginMarker singleton based on the fetch request. * The marker object is exported to the session using the result ticket specified in the request, * and flight info is returned containing the endpoint for accessing it. * - * Note: This exports PluginMarker.INSTANCE as a trusted marker. Plugin-specific routing + *

Note: This exports a PluginMarker for the specified plugin name. Plugin-specific routing * is handled by TypedTicket.type in the ConnectRequest phase, which is validated against * the plugin's name() method. * @@ -83,7 +82,7 @@ private static Any parseOrNull(final ByteString data) { * @param descriptor the flight descriptor containing the command * @param request the parsed RemoteFileSourcePluginFetchRequest containing the result ticket * @return a FlightInfo export object containing the plugin endpoint information - * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket + * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket or plugin name */ private static SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, @@ -91,23 +90,20 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { - throw new StatusRuntimeException(Status.INVALID_ARGUMENT - .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid result_id")); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "RemoteFileSourcePluginFetchRequest must contain a valid result_id"); } final String pluginName = request.getPluginName(); if (pluginName.isEmpty()) { - throw new StatusRuntimeException(Status.INVALID_ARGUMENT - .withDescription("RemoteFileSourcePluginFetchRequest must contain a valid plugin_name")); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, + "RemoteFileSourcePluginFetchRequest must contain a valid plugin_name"); } - final SessionState.ExportBuilder markerExportBuilder = - session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket"); - - // Get singleton marker for this plugin name - // This ensures isType() routing works correctly when multiple plugins use PluginMarker - final SessionState.ExportObject markerExport = - markerExportBuilder.submit(() -> PluginMarker.forPluginName(pluginName)); + // Export a plugin-specific PluginMarker. Plugins using PluginMarker should check + // marker.getPluginName() in isType() to prevent conflicts when multiple plugins share PluginMarker. + session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket") + .submit(() -> PluginMarker.forPluginName(pluginName)); final Flight.FlightInfo flightInfo = Flight.FlightInfo.newBuilder() .setFlightDescriptor(descriptor) @@ -119,6 +115,7 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl .setTotalRecords(-1) .setTotalBytes(-1) .build(); + return SessionState.wrapAsExport(flightInfo); } @@ -139,88 +136,130 @@ public SessionState.ExportObject flightInfoFor(@Nullable fina final Flight.FlightDescriptor descriptor, final String logId) { if (session == null) { - throw new StatusRuntimeException(Status.UNAUTHENTICATED); + throw Exceptions.statusRuntimeException(Code.UNAUTHENTICATED, + "Could not resolve '" + logId + "': no session available"); } - final Any request = parseOrNull(descriptor.getCmd()); + final Any request = parseAsAnyOrNull(descriptor.getCmd()); if (request == null) { log.error().append("Could not parse remotefilesource command.").endl(); throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Could not parse remotefilesource command Any."); } - if (FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { - return fetchPlugin(session, descriptor, parseFetchRequest(request)); + if (!FETCH_PLUGIN_TYPE_URL.equals(request.getTypeUrl())) { + log.error().append("Invalid remotefilesource command typeUrl: " + request.getTypeUrl()).endl(); + throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Invalid typeUrl: " + request.getTypeUrl()); } - log.error().append("Invalid pivot command typeUrl: " + request.getTypeUrl()).endl(); - throw Exceptions.statusRuntimeException(Code.INVALID_ARGUMENT, "Invalid typeUrl: " + request.getTypeUrl()); + return fetchPlugin(session, descriptor, parseFetchRequest(request)); } + /** + * Visits all flight info that this resolver exposes. + * + *

Not implemented: This resolver does not expose any flight info via list flights. + */ @Override public void forAllFlightInfo(@Nullable final SessionState session, final Consumer visitor) { // nothing to do } + /** + * Returns a log-friendly name for the given ticket. + * + * @throws UnsupportedOperationException always, as this resolver does not support ticket-based routing + */ @Override public String getLogNameFor(final ByteBuffer ticket, final String logId) { - // no tickets + // no ticket-based routing throw new UnsupportedOperationException(); } + /** + * Determines if this resolver is responsible for handling the given command descriptor. + * This resolver handles commands with type URL matching RemoteFileSourcePluginFetchRequest. + * + * @param descriptor the flight descriptor containing the command + * @return true if this resolver handles the command, false otherwise + */ @Override public boolean handlesCommand(final Flight.FlightDescriptor descriptor) { - // If not CMD, there is an error with io.deephaven.server.session.TicketRouter.getPathResolver / handlesPath Assert.eq(descriptor.getType(), "descriptor.getType()", Flight.FlightDescriptor.DescriptorType.CMD, "CMD"); - // No good way to check if this is a valid command without parsing to Any first. - final Any command = parseOrNull(descriptor.getCmd()); + final Any command = parseAsAnyOrNull(descriptor.getCmd()); if (command == null) { return false; } - // Check if the command matches any types that this resolver handles. return FETCH_PLUGIN_TYPE_URL.equals(command.getTypeUrl()); } + /** + * Publishes an export to the session using a ticket. + * + * @throws UnsupportedOperationException always, as this resolver does not support publishing + */ @Override public SessionState.ExportBuilder publish(final SessionState session, final ByteBuffer ticket, final String logId, @Nullable final Runnable onPublish) { - // no publishing throw new UnsupportedOperationException(); } + /** + * Publishes an export to the session using a flight descriptor. + * + * @throws UnsupportedOperationException always, as this resolver does not support publishing + */ @Override public SessionState.ExportBuilder publish(final SessionState session, final Flight.FlightDescriptor descriptor, final String logId, @Nullable final Runnable onPublish) { - // no publishing throw new UnsupportedOperationException(); } + /** + * Resolves a flight descriptor to an export object. + * + * @throws UnsupportedOperationException always, use flightInfoFor() instead + */ @Override public SessionState.ExportObject resolve(@Nullable final SessionState session, final Flight.FlightDescriptor descriptor, final String logId) { - // use flightInfoFor() instead of resolve() for descriptor handling throw new UnsupportedOperationException(); } + /** + * Resolves a ticket to an export object. + * + * @throws UnsupportedOperationException always, as this resolver does not support ticket-based routing + */ @Override public SessionState.ExportObject resolve(@Nullable final SessionState session, final ByteBuffer ticket, final String logId) { - // no tickets throw new UnsupportedOperationException(); } + /** + * Sets the ticket router for this resolver. + * + *

Not implemented: This resolver does not need access to the ticket router. + */ @Override public void setTicketRouter(TicketRouter ticketRouter) { // not needed } + /** + * Returns the ticket route byte for this resolver. + * This resolver does not use ticket-based routing, so returns 0. + * + * @return 0, indicating no ticket routing + */ @Override public byte ticketRoute() { return 0; From b553d6d7ac9e965b48ecd1396604605210554122 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:11:13 -0600 Subject: [PATCH 39/78] Cleanup RemoteFileSourceMessageStream (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 296 ++++++++++-------- .../proto/remotefilesource.proto | 3 - 2 files changed, 158 insertions(+), 141 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 4161d4a937d..85ed23d823c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -4,37 +4,42 @@ package io.deephaven.remotefilesource; import com.google.protobuf.InvalidProtocolBufferException; +import io.deephaven.engine.util.RemoteFileSourceClassLoader; +import io.deephaven.engine.util.RemoteFileSourceProvider; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectCommunicationException; import io.deephaven.plugin.type.ObjectType; import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; /** * Message stream implementation for RemoteFileSource bidirectional communication. * Each instance represents a file source provider for one client connection and implements - * RemoteFileSourceProvider so it can be registered with the ClassLoader. + * RemoteFileSourceProvider so it can be registered with the RemoteFileSourceClassLoader. * Only one MessageStream can be "active" at a time (determined by the execution context). - * The ClassLoader checks isActive() on each registered provider to find the active one. + * The RemoteFileSourceClassLoader checks isActive() on each registered provider to find the active one. */ -public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, io.deephaven.engine.util.RemoteFileSourceProvider { +public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, RemoteFileSourceProvider { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceMessageStream.class); /** * The current execution context containing the active message stream and configuration. * Null when no execution context is active. - * This is accessed by RemoteFileSourcePlugin.PROVIDER to route resource requests to the - * currently active message stream. + * Used by this class's isActive() and canSourceResource() methods to determine if this + * provider should handle resource requests from RemoteFileSourceClassLoader. */ private static volatile RemoteFileSourceExecutionContext executionContext; @@ -42,40 +47,48 @@ public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, private final ObjectType.MessageStream connection; private final Map> pendingRequests = new ConcurrentHashMap<>(); + /** + * Creates a new RemoteFileSourceMessageStream for the given connection. + * Automatically registers this instance as a provider with the RemoteFileSourceClassLoader. + * + * @param connection the message stream connection to the client + */ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) { this.connection = connection; - // Register this instance as a provider with the ClassLoader + // Register this instance as a provider with the RemoteFileSourceClassLoader registerWithClassLoader(); } - // RemoteFileSourceProvider interface implementation - each instance is a provider - + /** + * Determines if this provider can source the specified resource. + * Only returns true if this message stream is active, the resource is a .groovy file, + * and the resource path matches one of the configured resource paths. + * + * @param resourcePath the path of the resource to check + * @return true if this provider can source the resource, false otherwise + */ @Override - public boolean canSourceResource(String resourceName) { + public boolean canSourceResource(String resourcePath) { // Only active if this instance is the currently active message stream if (!isActive()) { return false; } // Only handle .groovy source files, not compiled .class files - if (!resourceName.endsWith(".groovy")) { + if (!resourcePath.endsWith(".groovy")) { return false; } RemoteFileSourceExecutionContext context = executionContext; - if (context == null || context.getActiveMessageStream() != this) { - return false; - } - java.util.List resourcePaths = context.getResourcePaths(); + List resourcePaths = context.getResourcePaths(); if (resourcePaths.isEmpty()) { return false; } - // Resource names from ClassLoader always use forward slashes, not backslashes for (String contextResourcePath : resourcePaths) { - if (resourceName.equals(contextResourcePath)) { - log.info().append("✅ Can source: ").append(resourceName).endl(); + if (resourcePath.equals(contextResourcePath)) { + log.info().append("Can source: ").append(resourcePath).endl(); return true; } } @@ -83,38 +96,46 @@ public boolean canSourceResource(String resourceName) { return false; } + /** + * Requests a resource from the remote client. + * Sends a request to the client and returns a future that will be completed when the client responds. + * Only services requests if this message stream is active. + * + * @param resourcePath the name of the resource to request + * @return a CompletableFuture that will contain the resource bytes when available, or null if inactive + */ @Override - public java.util.concurrent.CompletableFuture requestResource(String resourceName) { + public CompletableFuture requestResource(String resourcePath) { // Only service requests if this instance is active if (!isActive()) { - log.warn().append("Request for resource ").append(resourceName) + log.warn().append("Request for resource ").append(resourcePath) .append(" on inactive message stream").endl(); - return java.util.concurrent.CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(null); } - log.info().append("📥 Requesting resource: ").append(resourceName).endl(); + log.info().append("Requesting resource: ").append(resourcePath).endl(); - String requestId = java.util.UUID.randomUUID().toString(); - java.util.concurrent.CompletableFuture future = new java.util.concurrent.CompletableFuture<>(); + String requestId = UUID.randomUUID().toString(); + CompletableFuture future = new CompletableFuture<>(); pendingRequests.put(requestId, future); try { // Build RemoteFileSourceMetaRequest proto - io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest metaRequest = - io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourceName) + RemoteFileSourceMetaRequest metaRequest = + RemoteFileSourceMetaRequest.newBuilder() + .setResourceName(resourcePath) .build(); // Wrap in RemoteFileSourceServerRequest (server→client) - io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest message = - io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest.newBuilder() + RemoteFileSourceServerRequest message = + RemoteFileSourceServerRequest.newBuilder() .setRequestId(requestId) .setMetaRequest(metaRequest) .build(); - java.nio.ByteBuffer buffer = java.nio.ByteBuffer.wrap(message.toByteArray()); + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); - log.info().append("Sending resource request for: ").append(resourceName) + log.info().append("Sending resource request for: ").append(resourcePath) .append(" with requestId: ").append(requestId).endl(); connection.onData(buffer); @@ -126,30 +147,34 @@ public java.util.concurrent.CompletableFuture requestResource(String res return future; } + /** + * Checks if this message stream is currently active. + * A message stream is active when the execution context is set and this instance is the active stream. + * + * @return true if this message stream is active, false otherwise + */ @Override public boolean isActive() { RemoteFileSourceExecutionContext context = executionContext; return context != null && context.getActiveMessageStream() == this; } - // Static methods for execution context management - /** * Sets the execution context with the active message stream and resource paths. * This should be called when a script execution begins. * - * @param messageStream the message stream to set as active (must not be null) - * @param packages list of resource paths to resolve from remote source - * @throws IllegalArgumentException if messageStream is null (use clearExecutionContext() instead) + * @param messageStream the message stream to set as active + * @param resourcePaths list of resource paths to resolve from remote source + * @throws IllegalArgumentException if messageStream is null */ - public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, java.util.List packages) { + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, List resourcePaths) { if (messageStream == null) { - throw new IllegalArgumentException("messageStream must not be null. Use clearExecutionContext() to clear the context."); + throw new IllegalArgumentException("messageStream must not be null"); } - executionContext = new RemoteFileSourceExecutionContext(messageStream, packages); + executionContext = new RemoteFileSourceExecutionContext(messageStream, resourcePaths); log.info().append("Set execution context with ") - .append(packages != null ? packages.size() : 0).append(" resource paths").endl(); + .append(executionContext.getResourcePaths().size()).append(" resource paths").endl(); } /** @@ -171,70 +196,26 @@ public static RemoteFileSourceExecutionContext getExecutionContext() { return executionContext; } - // Instance methods for MessageStream implementation - + /** + * Handles incoming data from the client. + * Parses RemoteFileSourceClientRequest messages and processes meta responses + * or execution context updates from the client. + * + * @param payload the message payload containing the protobuf data + * @param references optional references (not used) + * @throws ObjectCommunicationException if the message cannot be parsed + */ @Override public void onData(ByteBuffer payload, Object... references) throws ObjectCommunicationException { try { - // Parse as RemoteFileSourceClientRequest proto (client→server) byte[] bytes = new byte[payload.remaining()]; payload.get(bytes); RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); - String requestId = message.getRequestId(); - if (message.hasMetaResponse()) { - // Client is responding to a resource request - RemoteFileSourceMetaResponse response = message.getMetaResponse(); - - CompletableFuture future = pendingRequests.remove(requestId); - if (future != null) { - byte[] content = response.getContent().toByteArray(); - - log.info().append("Received resource response for requestId: ").append(requestId) - .append(", found: ").append(response.getFound()) - .append(", content length: ").append(content.length).endl(); - - if (!response.getError().isEmpty()) { - log.warn().append("Error in response: ").append(response.getError()).endl(); - } - - future.complete(content); - } else { - log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); - } - } else if (message.hasTestCommand()) { - // Client sent a test command - String command = message.getTestCommand(); - log.info().append("Received test command from client: ").append(command).endl(); - - if (command.startsWith("TEST:")) { - String resourceName = command.substring(5).trim(); - log.info().append("Client initiated test for resource: ").append(resourceName).endl(); - testRequestResource(resourceName); - } + handleMetaResponse(message.getRequestId(), message.getMetaResponse()); } else if (message.hasSetExecutionContext()) { - // Client is requesting this message stream to become active - java.util.List packages = message.getSetExecutionContext().getResourcePathsList(); - setExecutionContext(this, packages); - log.info().append("Client set execution context for this message stream with ") - .append(packages.size()).append(" resource paths").endl(); - - // Send acknowledgment back to client - SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() - .setSuccess(true) - .build(); - - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setSetExecutionContextResponse(response) - .build(); - - try { - connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); - } catch (ObjectCommunicationException e) { - log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); - } + handleSetExecutionContext(message.getRequestId(), message.getSetExecutionContext().getResourcePathsList()); } else { log.warn().append("Received unknown message type from client").endl(); } @@ -244,14 +225,80 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } } + /** + * Handles a meta response from the client containing requested resource content. + * + * @param requestId the request ID + * @param response the meta response from the client + */ + private void handleMetaResponse(String requestId, RemoteFileSourceMetaResponse response) { + CompletableFuture future = pendingRequests.remove(requestId); + if (future == null) { + log.warn().append("Received response for unknown requestId: ").append(requestId).endl(); + return; + } + + byte[] content = response.getContent().toByteArray(); + + log.info().append("Received resource response for requestId: ").append(requestId) + .append(", found: ").append(response.getFound()) + .append(", content length: ").append(content.length).endl(); + + if (!response.getError().isEmpty()) { + log.warn().append("Error in response: ").append(response.getError()).endl(); + } + + future.complete(content); + } + + /** + * Handles a request from the client to set the execution context. + * + * @param requestId the request ID + * @param resourcePaths the list of resource paths to resolve from remote source + */ + private void handleSetExecutionContext(String requestId, List resourcePaths) { + setExecutionContext(this, resourcePaths); + log.info().append("Client set execution context for this message stream with ") + .append(resourcePaths.size()).append(" resource paths").endl(); + + sendExecutionContextAcknowledgment(requestId); + } + + /** + * Sends an acknowledgment to the client that the execution context was successfully set. + * + * @param requestId the request ID to acknowledge + */ + private void sendExecutionContextAcknowledgment(String requestId) { + SetExecutionContextResponse response = SetExecutionContextResponse.newBuilder() + .setSuccess(true) + .build(); + + RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + .setRequestId(requestId) + .setSetExecutionContextResponse(response) + .build(); + + try { + connection.onData(ByteBuffer.wrap(serverRequest.toByteArray())); + } catch (ObjectCommunicationException e) { + log.error().append("Failed to send execution context acknowledgment: ").append(e).endl(); + } + } + + /** + * Handles cleanup when the message stream is closed. + * Unregisters this provider from the RemoteFileSourceClassLoader, clears the execution context if this was active, + * and cancels all pending resource requests. + */ @Override public void onClose() { - // Unregister this provider from the ClassLoader + // Unregister this provider from the RemoteFileSourceClassLoader unregisterFromClassLoader(); // Clear execution context if this was the active stream - RemoteFileSourceExecutionContext context = executionContext; - if (context != null && context.getActiveMessageStream() == this) { + if (isActive()) { clearExecutionContext(); } @@ -261,58 +308,31 @@ public void onClose() { } /** - * Register this message stream instance as a provider with the ClassLoader. + * Register this message stream instance as a provider with the RemoteFileSourceClassLoader. */ private void registerWithClassLoader() { - io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = - io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); if (classLoader != null) { classLoader.registerProvider(this); - log.info().append("✅ Registered RemoteFileSourceMessageStream provider with ClassLoader").endl(); + log.info().append("Registered RemoteFileSourceMessageStream provider with RemoteFileSourceClassLoader").endl(); } else { - log.warn().append("⚠️ RemoteFileSourceClassLoader not available").endl(); + log.warn().append("RemoteFileSourceClassLoader not available").endl(); } } /** - * Unregister this message stream instance from the ClassLoader. + * Unregister this message stream instance from the RemoteFileSourceClassLoader. */ private void unregisterFromClassLoader() { - io.deephaven.engine.util.RemoteFileSourceClassLoader classLoader = - io.deephaven.engine.util.RemoteFileSourceClassLoader.getInstance(); + RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); if (classLoader != null) { classLoader.unregisterProvider(this); - log.info().append("🔴 Unregistered RemoteFileSourceMessageStream provider from ClassLoader").endl(); + log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); } } - /** - * Test method to request a resource and log the result. This can be called from the server console to test the - * bidirectional communication. - * - * @param resourceName the resource to request - */ - public void testRequestResource(String resourceName) { - log.info().append("Testing resource request for: ").append(resourceName).endl(); - - requestResource(resourceName) - .orTimeout(30, TimeUnit.SECONDS) - .whenComplete((content, error) -> { - if (error != null) { - log.error().append("Error requesting resource ").append(resourceName) - .append(": ").append(error).endl(); - } else { - log.info().append("Successfully received resource ").append(resourceName) - .append(" (").append(content.length).append(" bytes)").endl(); - if (content.length > 0 && content.length < 1000) { - String contentStr = new String(content, StandardCharsets.UTF_8); - log.info().append("Resource content:\n").append(contentStr).endl(); - } - } - }); - } /** * Encapsulates the execution context for remote file source operations. @@ -322,7 +342,7 @@ public void testRequestResource(String resourceName) { */ public static class RemoteFileSourceExecutionContext { private final RemoteFileSourceMessageStream activeMessageStream; - private final java.util.List resourcePaths; + private final List resourcePaths; /** * Creates a new execution context. @@ -331,9 +351,9 @@ public static class RemoteFileSourceExecutionContext { * @param resourcePaths list of resource paths to resolve from remote source */ public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - java.util.List resourcePaths) { + List resourcePaths) { this.activeMessageStream = activeMessageStream; - this.resourcePaths = resourcePaths != null ? resourcePaths : java.util.Collections.emptyList(); + this.resourcePaths = resourcePaths != null ? resourcePaths : Collections.emptyList(); } /** @@ -350,8 +370,8 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { * * @return a copy of the list of resource paths */ - public java.util.List getResourcePaths() { - return new java.util.ArrayList<>(resourcePaths); + public List getResourcePaths() { + return new ArrayList<>(resourcePaths); } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index b383a776e6b..26cc68c9986 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -36,9 +36,6 @@ message RemoteFileSourceClientRequest { // Set the execution context ID for script execution SetExecutionContextRequest set_execution_context = 3; - - // Test command (e.g., "TEST:com/example/Test.java") - client triggers server to request a resource back - string test_command = 4; } } From 86df3e085963795764506d6d60e0917e40c9c363 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:23:36 -0600 Subject: [PATCH 40/78] Cleanup RemoteFileSourcePlugin (#DH-20578) --- .../RemoteFileSourcePlugin.java | 28 +++++++++---------- .../JsRemoteFileSourceService.java | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 09af2b3430f..92107412fba 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -4,8 +4,6 @@ package io.deephaven.remotefilesource; import com.google.auto.service.AutoService; -import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectType; import io.deephaven.plugin.type.ObjectTypeBase; import io.deephaven.plugin.type.ObjectCommunicationException; @@ -14,24 +12,24 @@ import java.nio.ByteBuffer; /** - * ObjectType plugin for RemoteFileSource. This plugin is registered via @AutoService - * and handles creation of RemoteFileSourceMessageStream connections. - * - * This plugin uses a PluginMarker with a type field instead of instanceof checks, - * allowing it to work across language boundaries (Java/Python). The Flight command - * creates a PluginMarker with type="RemoteFileSource" which this plugin recognizes. - * + * ObjectType plugin for remote file sources. This plugin is registered via @AutoService + * and handles creation of RemoteFileSourceMessageStream connections for bidirectional + * communication with clients. + *

+ * This plugin recognizes PluginMarker objects whose pluginName matches this plugin's name. + * When a connection is established, a RemoteFileSourceMessageStream is created to handle + * bidirectional message passing between client and server. + *

* Each RemoteFileSourceMessageStream instance registers itself as a provider with the - * ClassLoader when created and unregisters when closed. The ClassLoader checks isActive() - * on each registered provider to find the currently active one. + * RemoteFileSourceClassLoader when created and unregisters when closed. The RemoteFileSourceClassLoader + * uses isActive() to determine which registered provider should handle resource requests. */ @AutoService(ObjectType.class) public class RemoteFileSourcePlugin extends ObjectTypeBase { - private static final Logger log = LoggerFactory.getLogger(RemoteFileSourcePlugin.class); @Override public String name() { - return "DeephavenGroovyRemoteFileSourcePlugin"; + return "DeephavenRemoteFileSourcePlugin"; } @Override @@ -50,10 +48,10 @@ public MessageStream compatibleClientConnection(Object object, MessageStream con throw new ObjectCommunicationException("Expected RemoteFileSource marker object, got " + object.getClass()); } + // Send initial empty message to client as required by the ObjectType contract connection.onData(ByteBuffer.allocate(0)); - // Create and return a new message stream for this connection - // All the logic is in the static RemoteFileSourceMessageStream class + // Return a new bidirectional message stream for this connection return new RemoteFileSourceMessageStream(connection); } } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index dd0d6ef0f09..f5bc526cca1 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -104,7 +104,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenGroovyRemoteFileSourcePlugin"; + String pluginName = "DeephavenRemoteFileSourcePlugin"; return fetchPluginFlightInfo(connection, pluginName) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. From ce122c502b0a9b5aeef43ca589695b8b672def83 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:25:15 -0600 Subject: [PATCH 41/78] Cleanup RemoteFileSourceTicketResolverFactoryService (#DH-20578) --- .../RemoteFileSourceTicketResolverFactoryService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java index fa398c85d95..204161b28a4 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -7,6 +7,11 @@ import io.deephaven.server.runner.TicketResolversFromServiceLoader; import io.deephaven.server.session.TicketResolver; +/** + * Factory service for creating RemoteFileSourceCommandResolver instances. + * This service is registered via @AutoService and provides ticket resolver functionality + * for handling remote file source plugin commands through the Deephaven server infrastructure. + */ @AutoService(TicketResolversFromServiceLoader.Factory.class) public class RemoteFileSourceTicketResolverFactoryService implements TicketResolversFromServiceLoader.Factory { @Override From 6b284d69784a3c1b99e4b8521e31cf0248e36131 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:35:06 -0600 Subject: [PATCH 42/78] Cleanup PluginMarker (#DH-20578) --- .../main/java/io/deephaven/plugin/type/PluginMarker.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 570af422ecc..158ff2d0a65 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -3,16 +3,21 @@ // package io.deephaven.plugin.type; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + /** * A generic marker object for plugin exports that can be shared across multiple plugin types. + *

* IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. + *

* This class uses a singleton pattern - one instance per pluginName. */ public class PluginMarker { - private static final java.util.Map INSTANCES = new java.util.concurrent.ConcurrentHashMap<>(); + private static final Map INSTANCES = new ConcurrentHashMap<>(); private final String pluginName; From d42d61fd15e95c728bbd9d06acbc678db039e2c7 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:44:13 -0600 Subject: [PATCH 43/78] Changed back to resourceName convention to match class loaders (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 22 +++++++++---------- .../proto/remotefilesource.proto | 6 ++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 85ed23d823c..9cc3879b80f 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -64,18 +64,18 @@ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) * Only returns true if this message stream is active, the resource is a .groovy file, * and the resource path matches one of the configured resource paths. * - * @param resourcePath the path of the resource to check + * @param resourceName the name of the resource to check * @return true if this provider can source the resource, false otherwise */ @Override - public boolean canSourceResource(String resourcePath) { + public boolean canSourceResource(String resourceName) { // Only active if this instance is the currently active message stream if (!isActive()) { return false; } // Only handle .groovy source files, not compiled .class files - if (!resourcePath.endsWith(".groovy")) { + if (!resourceName.endsWith(".groovy")) { return false; } @@ -87,8 +87,8 @@ public boolean canSourceResource(String resourcePath) { } for (String contextResourcePath : resourcePaths) { - if (resourcePath.equals(contextResourcePath)) { - log.info().append("Can source: ").append(resourcePath).endl(); + if (resourceName.equals(contextResourcePath)) { + log.info().append("Can source: ").append(resourceName).endl(); return true; } } @@ -101,19 +101,19 @@ public boolean canSourceResource(String resourcePath) { * Sends a request to the client and returns a future that will be completed when the client responds. * Only services requests if this message stream is active. * - * @param resourcePath the name of the resource to request + * @param resourceName the name of the resource to request * @return a CompletableFuture that will contain the resource bytes when available, or null if inactive */ @Override - public CompletableFuture requestResource(String resourcePath) { + public CompletableFuture requestResource(String resourceName) { // Only service requests if this instance is active if (!isActive()) { - log.warn().append("Request for resource ").append(resourcePath) + log.warn().append("Request for resource ").append(resourceName) .append(" on inactive message stream").endl(); return CompletableFuture.completedFuture(null); } - log.info().append("Requesting resource: ").append(resourcePath).endl(); + log.info().append("Requesting resource: ").append(resourceName).endl(); String requestId = UUID.randomUUID().toString(); CompletableFuture future = new CompletableFuture<>(); @@ -123,7 +123,7 @@ public CompletableFuture requestResource(String resourcePath) { // Build RemoteFileSourceMetaRequest proto RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourcePath) + .setResourceName(resourceName) .build(); // Wrap in RemoteFileSourceServerRequest (server→client) @@ -135,7 +135,7 @@ public CompletableFuture requestResource(String resourcePath) { ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); - log.info().append("Sending resource request for: ").append(resourcePath) + log.info().append("Sending resource request for: ").append(resourceName) .append(" with requestId: ").append(requestId).endl(); connection.onData(buffer); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 26cc68c9986..20cd589fce1 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending + * Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending */ syntax = "proto3"; @@ -34,7 +34,7 @@ message RemoteFileSourceClientRequest { // Response to a resource request RemoteFileSourceMetaResponse meta_response = 2; - // Set the execution context ID for script execution + // Set the execution context for script execution SetExecutionContextRequest set_execution_context = 3; } } @@ -53,7 +53,7 @@ message RemoteFileSourceMetaResponse { // Indicates whether the resource was found bool found = 2; - // Optional: error message if the resource could not be retrieved + // Error message if the resource could not be retrieved string error = 3; } From 3856646d04477fce894232611b6c8ded5d6665a8 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 15:52:47 -0600 Subject: [PATCH 44/78] Moved method (#DH-20578) --- .../main/java/io/deephaven/web/client/api/CoreClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 833610bd030..78ba535a24f 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -142,6 +142,10 @@ public Promise onConnected(@JsOptional Double timeoutInMillis) { return ideConnection.onConnected(); } + public Promise getRemoteFileSourceService() { + return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + } + public Promise getServerConfigValues() { return getConfigs( c -> ideConnection.connection.get().configServiceClient().getConfigurationConstants( @@ -155,10 +159,6 @@ public JsStorageService getStorageService() { return new JsStorageService(ideConnection.connection.get()); } - public Promise getRemoteFileSourceService() { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); - } - public Promise getAsIdeConnection() { return Promise.resolve(ideConnection); } From 22a8f257ffed2210e74f479b9cba1e562efb6c85 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 16:00:53 -0600 Subject: [PATCH 45/78] Cleanup JsProtobufUtils (#DH-20578) --- .../web/client/api/JsProtobufUtils.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java index 7413aedd954..2d61a93523b 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java @@ -11,6 +11,11 @@ */ public class JsProtobufUtils { + // Varint encoding constants + private static final int VARINT_CONTINUATION_BIT = 0x80; // 10000000 - indicates more bytes follow + private static final int VARINT_DATA_MASK = 0x7F; // 01111111 - extracts 7 bits of data + private static final int VARINT_BYTE_THRESHOLD = 128; // Values >= 128 require multiple bytes + private JsProtobufUtils() { // Utility class, no instantiation } @@ -88,8 +93,8 @@ private static int calculateFieldSize(int tag, int dataLength) { *

  • 2 bytes: 128 to 16,383 (2^14 - 1)
    • *
  • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
    • *
  • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
    • - *
  • 5 bytes: 268,435,456 to 4,294,967,295 (2^35 - 1, max unsigned 32-bit)
    • - *
  • 10 bytes: negative numbers (due to sign extension)
    • + *
  • 5 bytes: 268,435,456 to 4,294,967,295 (max unsigned 32-bit int)
    • + *
  • 10 bytes: negative numbers (sign-extended to 64 bits in varint encoding)
    • * * * @param value the integer value to encode @@ -97,16 +102,16 @@ private static int calculateFieldSize(int tag, int dataLength) { */ private static int sizeOfVarint(int value) { if (value < 0) - return 10; // Negative numbers use sign extension, always 10 bytes - if (value < 128) // 2^7 + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < VARINT_BYTE_THRESHOLD) // 2^7 = 128 return 1; - if (value < 16384) // 2^14 + if (value < 16384) // 2^14 return 2; - if (value < 2097152) // 2^21 + if (value < 2097152) // 2^21 return 3; - if (value < 268435456) // 2^28 + if (value < 268435456) // 2^28 return 4; - return 5; // 2^35 (max for positive 32-bit int) + return 5; // Max unsigned 32-bit int requires 5 bytes } /** @@ -164,9 +169,9 @@ private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array da * @return the new position after writing */ private static int writeVarint(Uint8Array buffer, int pos, int value) { - while (value >= 128) { + while (value >= VARINT_BYTE_THRESHOLD) { // Extract lowest 7 bits and set continuation flag (8th bit = 1) - buffer.setAt(pos++, (double) ((value & 0x7F) | 0x80)); + buffer.setAt(pos++, (double) ((value & VARINT_DATA_MASK) | VARINT_CONTINUATION_BIT)); // Shift right by 7 to process next chunk value >>>= 7; // Unsigned right shift to handle large positive values } From 85431553ce64f768a55b22a38d557a75acfab83f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 2 Jan 2026 17:07:47 -0600 Subject: [PATCH 46/78] Cleanup JsRemoteFileSourceService (#DH-20578) --- .../JsRemoteFileSourceService.java | 157 +++++++++++++----- 1 file changed, 111 insertions(+), 46 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index f5bc526cca1..b1070ca3f60 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -43,13 +43,28 @@ /** * JavaScript client for the RemoteFileSource service. Provides bidirectional communication with the server-side - * RemoteFileSourceServicePlugin via a message stream. + * RemoteFileSourcePlugin via a message stream. + *

    + * Events: + *

      + *
    • {@link #EVENT_MESSAGE}: Fired for unrecognized messages from the server
      • + *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client
      • + *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { + /** Event name for generic messages from the server */ public static final String EVENT_MESSAGE = "message"; + + /** Event name for resource request events from the server */ public static final String EVENT_REQUEST_SOURCE = "requestsource"; + // Plugin name must match RemoteFileSourcePlugin.name() on the server + private static final String PLUGIN_NAME = "DeephavenRemoteFileSourcePlugin"; + + // Timeout for setExecutionContext requests (in milliseconds) + private static final int SET_EXECUTION_CONTEXT_TIMEOUT_MS = 30000; // 30 seconds + private final JsWidget widget; // Track pending setExecutionContext requests @@ -65,18 +80,17 @@ private JsRemoteFileSourceService(JsWidget widget) { * Fetches the FlightInfo for the plugin fetch command. * * @param connection the worker connection to use - * @param pluginName the name of the plugin to fetch * @return a promise that resolves to the FlightInfo for the plugin fetch */ @JsIgnore - private static Promise fetchPluginFlightInfo(WorkerConnection connection, String pluginName) { + private static Promise fetchPluginFlightInfo(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); // Create the fetch request RemoteFileSourcePluginFetchRequest fetchRequest = new RemoteFileSourcePluginFetchRequest(); fetchRequest.setResultId(resultTicket); - fetchRequest.setPluginName(pluginName); + fetchRequest.setPluginName(PLUGIN_NAME); // Serialize the request to bytes Uint8Array innerRequestBytes = fetchRequest.serializeBinary(); @@ -92,8 +106,8 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec descriptor.setCmd(anyWrappedBytes); // Send the getFlightInfo request - return Callbacks.grpcUnaryPromise( - c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); + return Callbacks.grpcUnaryPromise(c -> + connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); } /** @@ -104,8 +118,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec */ @JsIgnore public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { - String pluginName = "DeephavenRemoteFileSourcePlugin"; - return fetchPluginFlightInfo(connection, pluginName) + return fetchPluginFlightInfo(connection) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. // This is the standard Flight pattern: we passed resultTicket in the request, @@ -124,14 +137,14 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c // The type must match RemoteFileSourcePlugin.name() TypedTicket typedTicket = new TypedTicket(); typedTicket.setTicket(dhTicket); - typedTicket.setType(pluginName); + typedTicket.setType(PLUGIN_NAME); JsWidget widget = new JsWidget(connection, typedTicket); JsRemoteFileSourceService service = new JsRemoteFileSourceService(widget); return service.connect(); } else { - return Promise.reject("No endpoints returned from RemoteFileSource plugin fetch"); + return Promise.reject("No endpoints returned from " + PLUGIN_NAME + " plugin fetch"); } }); } @@ -143,47 +156,83 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c */ @JsIgnore private Promise connect() { - widget.addEventListener("message", (Event event) -> { - // Parse the message as RemoteFileSourceServerRequest proto (server→client) - Uint8Array payload = event.getDetail().getDataAsU8(); - - try { - RemoteFileSourceServerRequest message = - RemoteFileSourceServerRequest.deserializeBinary(payload); - - if (message.hasMetaRequest()) { - // If server has requested a resource from the client, fire request event - RemoteFileSourceMetaRequest request = message.getMetaRequest(); - - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, - new ResourceRequestEvent(message.getRequestId(), request)), 0); - } else if (message.hasSetExecutionContextResponse()) { - // Server acknowledged execution context was set - String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetExecutionContextRequests.remove(requestId); - if (resolveCallback != null) { - SetExecutionContextResponse response = message.getSetExecutionContextResponse(); - resolveCallback.onInvoke(response.getSuccess()); - } - } else { - // Unknown message type - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); - } - } catch (Exception e) { - // Failed to parse as proto, fire generic message event - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); - } - }); - + widget.addEventListener("message", this::handleMessage); return widget.refetch().then(w -> Promise.resolve(this)); } + /** + * Handles incoming messages from the server. + * + * @param event the message event from the server + */ + @JsIgnore + private void handleMessage(Event event) { + Uint8Array payload = event.getDetail().getDataAsU8(); + + RemoteFileSourceServerRequest message; + try { + message = RemoteFileSourceServerRequest.deserializeBinary(payload); + } catch (Exception e) { + // Failed to parse as proto, fire generic message event + handleUnknownMessage(event); + return; + } + + // Route the parsed message to the appropriate handler + if (message.hasMetaRequest()) { + handleMetaRequest(message); + } else if (message.hasSetExecutionContextResponse()) { + handleSetExecutionContextResponse(message); + } else { + handleUnknownMessage(event); + } + } + + /** + * Handles a meta request (resource request) from the server. + * + * @param message the server request message + */ + @JsIgnore + private void handleMetaRequest(RemoteFileSourceServerRequest message) { + RemoteFileSourceMetaRequest request = message.getMetaRequest(); + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, + new ResourceRequestEvent(message.getRequestId(), request)), 0); + } + + /** + * Handles a set execution context response from the server. + * + * @param message the server request message + */ + @JsIgnore + private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest message) { + String requestId = message.getRequestId(); + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = + pendingSetExecutionContextRequests.remove(requestId); + if (resolveCallback != null) { + SetExecutionContextResponse response = message.getSetExecutionContextResponse(); + resolveCallback.onInvoke(response.getSuccess()); + } + } + + /** + * Handles an unknown or unparseable message from the server. + * + * @param event the message event + */ + @JsIgnore + private void handleUnknownMessage(Event event) { + DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); + } + /** * Sets the execution context on the server to identify this message stream as active * for script execution. * - * @param resourcePaths array of resource paths to resolve from remote source (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) + * @param resourcePaths array of resource paths to resolve from remote source + * (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]), + * or null/empty for no specific resources * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod @@ -195,6 +244,17 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) // Store the resolve callback to call when we get the acknowledgment pendingSetExecutionContextRequests.put(requestId, resolve); + // Set a timeout to reject the promise if no response is received + DomGlobal.setTimeout(ignore -> { + Promise.PromiseExecutorCallbackFn.ResolveCallbackFn callback = + pendingSetExecutionContextRequests.remove(requestId); + if (callback != null) { + // Request timed out - reject the promise + reject.onInvoke("setExecutionContext request timed out after " + + SET_EXECUTION_CONTEXT_TIMEOUT_MS + "ms"); + } + }, SET_EXECUTION_CONTEXT_TIMEOUT_MS); + RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); sendClientRequest(clientRequest); }); @@ -232,7 +292,8 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { // Serialize the protobuf message to bytes Uint8Array messageBytes = clientRequest.serializeBinary(); - // Send as Uint8Array (which is an ArrayBufferView, compatible with MessageUnion) + // Uint8Array is an ArrayBufferView, which is one of the MessageUnion types + // The unchecked cast is safe because MessageUnion accepts String | ArrayBuffer | ArrayBufferView widget.sendMessage(Js.uncheckedCast(messageBytes), null); } @@ -271,7 +332,11 @@ public String getResourceName() { /** * Responds to this resource request with the given content. * - * @param content the resource content (string or Uint8Array), or null if not found + * @param content the resource content as a String, Uint8Array, or null to indicate + * the resource was not found. If a String is provided, it will be + * UTF-8 encoded before being sent to the server. Uint8Array content + * is sent as-is. + * @throws IllegalArgumentException if content is not a String, Uint8Array, or null */ @JsMethod public void respond(@JsNullable Object content) { From de0a35961d7ce94ae5403e09f932efa6b163de40 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 5 Jan 2026 15:13:57 -0600 Subject: [PATCH 47/78] Applying Colin's client proto generation (#DH-20578) --- .../RemoteFileSourceClientRequest.java | 22 +------------------ .../RequestCase.java | 4 ++-- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 314b642900a..8ba1dcc4846 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -122,9 +122,6 @@ static RemoteFileSourceClientRequest.ToObjectReturnType create() { @JsProperty RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType getSetExecutionContext(); - @JsProperty - String getTestCommand(); - @JsProperty void setMetaResponse( RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType metaResponse); @@ -135,9 +132,6 @@ void setMetaResponse( @JsProperty void setSetExecutionContext( RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType setExecutionContext); - - @JsProperty - void setTestCommand(String testCommand); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -245,9 +239,6 @@ static RemoteFileSourceClientRequest.ToObjectReturnType0 create() { @JsProperty RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType getSetExecutionContext(); - @JsProperty - String getTestCommand(); - @JsProperty void setMetaResponse( RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType metaResponse); @@ -258,9 +249,6 @@ void setMetaResponse( @JsProperty void setSetExecutionContext( RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType setExecutionContext); - - @JsProperty - void setTestCommand(String testCommand); } public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); @@ -278,8 +266,6 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native void clearSetExecutionContext(); - public native void clearTestCommand(); - public native RemoteFileSourceMetaResponse getMetaResponse(); public native int getRequestCase(); @@ -288,14 +274,10 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native SetExecutionContextRequest getSetExecutionContext(); - public native String getTestCommand(); - public native boolean hasMetaResponse(); public native boolean hasSetExecutionContext(); - public native boolean hasTestCommand(); - public native Uint8Array serializeBinary(); public native void setMetaResponse(); @@ -308,9 +290,7 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native void setSetExecutionContext(SetExecutionContextRequest value); - public native void setTestCommand(String value); - public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(boolean includeInstance); -} +} \ No newline at end of file diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java index 46db67bea6b..a5a156433ed 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java @@ -11,5 +11,5 @@ name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest.RequestCase", namespace = JsPackage.GLOBAL) public class RequestCase { - public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT, TEST_COMMAND; -} + public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT; +} \ No newline at end of file From b88b4eb6693aeee494a13f0fa7de4ee6400d8c75 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 5 Jan 2026 16:23:49 -0600 Subject: [PATCH 48/78] Changed to runtime dependencies and fixed docs links (#DH-20578) --- server/jetty-app-11/build.gradle | 2 +- server/jetty-app/build.gradle | 2 +- .../client/api/remotefilesource/JsRemoteFileSourceService.java | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/jetty-app-11/build.gradle b/server/jetty-app-11/build.gradle index 3a4680c88ca..09baafd32a8 100644 --- a/server/jetty-app-11/build.gradle +++ b/server/jetty-app-11/build.gradle @@ -12,11 +12,11 @@ dependencies { implementation project(':server-jetty-11') implementation project(':extensions-flight-sql') - implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler + runtimeOnly project(':plugin-remotefilesource') runtimeOnly project(':log-to-slf4j') runtimeOnly project(':logback-print-stream-globals') runtimeOnly project(':logback-logbuffer') diff --git a/server/jetty-app/build.gradle b/server/jetty-app/build.gradle index 728cddc629a..ec03184a602 100644 --- a/server/jetty-app/build.gradle +++ b/server/jetty-app/build.gradle @@ -12,11 +12,11 @@ dependencies { implementation project(':server-jetty') implementation project(':extensions-flight-sql') - implementation project(':plugin-remotefilesource') implementation libs.dagger annotationProcessor libs.dagger.compiler + runtimeOnly project(':plugin-remotefilesource') runtimeOnly project(':log-to-slf4j') runtimeOnly project(':logback-print-stream-globals') runtimeOnly project(':logback-logbuffer') diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index b1070ca3f60..79512bf0f60 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -54,9 +54,11 @@ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { /** Event name for generic messages from the server */ + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_MESSAGE = "message"; /** Event name for resource request events from the server */ + @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST_SOURCE = "requestsource"; // Plugin name must match RemoteFileSourcePlugin.name() on the server From 7ee9121f07550ecba6bc1d2cb15d1f926efce181 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 12:47:51 -0600 Subject: [PATCH 49/78] Addressed review comments in RemoteFileSourceClassLoader (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 3add4e468cc..470b6c88767 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -6,6 +6,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.net.URLStreamHandler; @@ -78,15 +79,19 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { */ @Override protected URL findResource(String name) { - for (RemoteFileSourceProvider provider : providers) { - if (!provider.isActive() || !provider.canSourceResource(name)) { - continue; + RemoteFileSourceProvider provider = null; + for (RemoteFileSourceProvider candidate : providers) { + if (candidate.isActive() && candidate.canSourceResource(name)) { + provider = candidate; + break; } + } + if (provider != null) { try { return new URL(null, "remotefile://" + name, new RemoteFileURLStreamHandler(provider, name)); - } catch (java.net.MalformedURLException e) { - // Continue to next provider if URL creation fails + } catch (MalformedURLException e) { + // Fall through to parent if URL creation fails } } @@ -169,10 +174,12 @@ public void connect() throws IOException { /** * Returns an input stream that reads from this connection's resource. * - *

    This method ensures the connection is established before returning the stream. + *

    This method calls {@link #connect()} to ensure the connection is established and resource bytes are + * fetched from the provider. The method then verifies that content has been successfully downloaded before + * creating the input stream. * - * @return an input stream that reads from the resource - * @throws IOException if the connection cannot be established or if the resource has no content + * @return an input stream that reads from the fetched resource bytes + * @throws IOException if the connection or content download fails or if the resource has no content */ @Override public InputStream getInputStream() throws IOException { From a8b8c619322511236a3305a6e65bec32e2bce4fd Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 12:57:44 -0600 Subject: [PATCH 50/78] Addressed review comments in PluginMarker (#DH-20578) --- .../main/java/io/deephaven/plugin/type/PluginMarker.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 158ff2d0a65..2eaa3dcb2f8 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -14,7 +14,8 @@ * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered * first would intercept all PluginMarker instances. *

    - * This class uses a singleton pattern - one instance per pluginName. + * This class maintains a single instance per pluginName - multiple calls to {@link #forPluginName(String)} + * with the same name will return the same instance. */ public class PluginMarker { private static final Map INSTANCES = new ConcurrentHashMap<>(); @@ -22,7 +23,7 @@ public class PluginMarker { private final String pluginName; /** - * Private constructor - use forPluginName() to get singleton instances. + * Private constructor - use forPluginName() to get instances. * * @param pluginName the plugin name identifier (should match the plugin's name() method) */ @@ -31,10 +32,10 @@ private PluginMarker(String pluginName) { } /** - * Gets the singleton PluginMarker instance for the specified plugin name. + * Gets the PluginMarker instance for the specified plugin name, creating it if necessary. * * @param pluginName the plugin name identifier (should match the plugin's name() method) - * @return the singleton PluginMarker for this plugin name + * @return the PluginMarker instance for this plugin name * @throws IllegalArgumentException if pluginName is null or empty */ public static PluginMarker forPluginName(String pluginName) { From accb2dc58140b5ed5ebca396637d94d06ef31f12 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 13:08:16 -0600 Subject: [PATCH 51/78] Addressed review comments in RemoteFileSourceMessageStream (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 9cc3879b80f..cbafb2e1dc4 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -17,8 +17,6 @@ import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; @@ -88,7 +86,7 @@ public boolean canSourceResource(String resourceName) { for (String contextResourcePath : resourcePaths) { if (resourceName.equals(contextResourcePath)) { - log.info().append("Can source: ").append(resourceName).endl(); + log.debug().append("Can source: ").append(resourceName).endl(); return true; } } @@ -187,15 +185,6 @@ public static void clearExecutionContext() { } } - /** - * Gets the current execution context. - * - * @return the execution context - */ - public static RemoteFileSourceExecutionContext getExecutionContext() { - return executionContext; - } - /** * Handles incoming data from the client. * Parses RemoteFileSourceClientRequest messages and processes meta responses @@ -353,7 +342,7 @@ public static class RemoteFileSourceExecutionContext { public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, List resourcePaths) { this.activeMessageStream = activeMessageStream; - this.resourcePaths = resourcePaths != null ? resourcePaths : Collections.emptyList(); + this.resourcePaths = resourcePaths; } /** @@ -368,10 +357,10 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { /** * Gets the resource paths that should be resolved from the remote source. * - * @return a copy of the list of resource paths + * @return the list of resource paths */ public List getResourcePaths() { - return new ArrayList<>(resourcePaths); + return resourcePaths; } } } From e79b25f053a675644bcd5752d9ed1796658c6125 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 6 Jan 2026 15:33:28 -0600 Subject: [PATCH 52/78] Addressed review comments in JsRemoteFileSourceService (#DH-20578) --- .../JsRemoteFileSourceService.java | 84 +++++++------------ .../ResourceContentUnion.java | 49 +++++++++++ 2 files changed, 81 insertions(+), 52 deletions(-) create mode 100644 web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 79512bf0f60..957ee628c4b 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -5,7 +5,6 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; -import com.vertispan.tsdefs.annotations.TsTypeRef; import elemental2.core.Uint8Array; import elemental2.dom.DomGlobal; import elemental2.dom.TextEncoder; @@ -28,6 +27,7 @@ import io.deephaven.web.client.api.event.HasEventHandling; import io.deephaven.web.client.api.widget.JsWidget; import io.deephaven.web.client.api.widget.WidgetMessageDetails; +import io.deephaven.web.client.fu.LazyPromise; import jsinterop.annotations.JsIgnore; import jsinterop.annotations.JsMethod; import jsinterop.annotations.JsNullable; @@ -70,10 +70,9 @@ public class JsRemoteFileSourceService extends HasEventHandling { private final JsWidget widget; // Track pending setExecutionContext requests - private final Map> pendingSetExecutionContextRequests = new HashMap<>(); + private final Map> pendingSetExecutionContextRequests = new HashMap<>(); private int requestIdCounter = 0; - @JsIgnore private JsRemoteFileSourceService(JsWidget widget) { this.widget = widget; } @@ -84,7 +83,6 @@ private JsRemoteFileSourceService(JsWidget widget) { * @param connection the worker connection to use * @return a promise that resolves to the FlightInfo for the plugin fetch */ - @JsIgnore private static Promise fetchPluginFlightInfo(WorkerConnection connection) { // Create a new export ticket for the result Ticket resultTicket = connection.getTickets().newExportTicket(); @@ -119,7 +117,7 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec * @return a promise that resolves to a RemoteFileSourceService instance with an active message stream */ @JsIgnore - public static Promise fetchPlugin(@TsTypeRef(Object.class) WorkerConnection connection) { + public static Promise fetchPlugin(WorkerConnection connection) { return fetchPluginFlightInfo(connection) .then(flightInfo -> { // The first endpoint contains the ticket for the plugin instance. @@ -156,7 +154,6 @@ public static Promise fetchPlugin(@TsTypeRef(Object.c * * @return a promise that resolves to this service instance when the connection is established */ - @JsIgnore private Promise connect() { widget.addEventListener("message", this::handleMessage); return widget.refetch().then(w -> Promise.resolve(this)); @@ -167,7 +164,6 @@ private Promise connect() { * * @param event the message event from the server */ - @JsIgnore private void handleMessage(Event event) { Uint8Array payload = event.getDetail().getDataAsU8(); @@ -195,7 +191,6 @@ private void handleMessage(Event event) { * * @param message the server request message */ - @JsIgnore private void handleMetaRequest(RemoteFileSourceServerRequest message) { RemoteFileSourceMetaRequest request = message.getMetaRequest(); DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, @@ -207,14 +202,12 @@ private void handleMetaRequest(RemoteFileSourceServerRequest message) { * * @param message the server request message */ - @JsIgnore private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest message) { String requestId = message.getRequestId(); - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn resolveCallback = - pendingSetExecutionContextRequests.remove(requestId); - if (resolveCallback != null) { + LazyPromise promise = pendingSetExecutionContextRequests.remove(requestId); + if (promise != null) { SetExecutionContextResponse response = message.getSetExecutionContextResponse(); - resolveCallback.onInvoke(response.getSuccess()); + promise.succeed(response.getSuccess()); } } @@ -223,7 +216,6 @@ private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest mes * * @param event the message event */ - @JsIgnore private void handleUnknownMessage(Event event) { DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); } @@ -239,27 +231,19 @@ private void handleUnknownMessage(Event event) { */ @JsMethod public Promise setExecutionContext(@JsOptional String[] resourcePaths) { - return new Promise<>((resolve, reject) -> { - // Generate a unique request ID - String requestId = "setExecutionContext-" + (requestIdCounter++); - - // Store the resolve callback to call when we get the acknowledgment - pendingSetExecutionContextRequests.put(requestId, resolve); - - // Set a timeout to reject the promise if no response is received - DomGlobal.setTimeout(ignore -> { - Promise.PromiseExecutorCallbackFn.ResolveCallbackFn callback = - pendingSetExecutionContextRequests.remove(requestId); - if (callback != null) { - // Request timed out - reject the promise - reject.onInvoke("setExecutionContext request timed out after " - + SET_EXECUTION_CONTEXT_TIMEOUT_MS + "ms"); - } - }, SET_EXECUTION_CONTEXT_TIMEOUT_MS); + // Generate a unique request ID + String requestId = "setExecutionContext-" + (requestIdCounter++); - RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); - sendClientRequest(clientRequest); - }); + // Create a lazy promise that will be resolved when we get the response + LazyPromise promise = new LazyPromise<>(); + pendingSetExecutionContextRequests.put(requestId, promise); + + // Send the request + RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); + sendClientRequest(clientRequest); + + // Return a promise with built-in timeout + return promise.asPromise(SET_EXECUTION_CONTEXT_TIMEOUT_MS); } /** @@ -273,9 +257,7 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); if (resourcePaths != null) { - for (String resourcePath : resourcePaths) { - setContextRequest.addResourcePaths(resourcePath); - } + setContextRequest.setResourcePathsList(resourcePaths); } RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); @@ -289,7 +271,6 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) * * @param clientRequest the client request to send */ - @JsIgnore private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { // Serialize the protobuf message to bytes Uint8Array messageBytes = clientRequest.serializeBinary(); @@ -302,7 +283,6 @@ private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { /** * Closes the message stream connection to the server. */ - @JsMethod public void close() { widget.close(); } @@ -312,12 +292,11 @@ public void close() { * respond() method. */ @TsInterface - @TsName(namespace = "dh.remotefilesource", name = "ResourceRequestEvent") + @TsName(namespace = "dh.remotefilesource") public class ResourceRequestEvent { private final String requestId; private final RemoteFileSourceMetaRequest protoRequest; - @JsIgnore public ResourceRequestEvent(String requestId, RemoteFileSourceMetaRequest protoRequest) { this.requestId = requestId; this.protoRequest = protoRequest; @@ -334,14 +313,15 @@ public String getResourceName() { /** * Responds to this resource request with the given content. * - * @param content the resource content as a String, Uint8Array, or null to indicate - * the resource was not found. If a String is provided, it will be - * UTF-8 encoded before being sent to the server. Uint8Array content - * is sent as-is. - * @throws IllegalArgumentException if content is not a String, Uint8Array, or null + * @param content the resource content (String | Uint8Array | null): + *

      + *
    • String - will be UTF-8 encoded before sending to server
      • + *
    • Uint8Array - sent as-is to server
      • + *
    • null - indicates resource was not found
      • + *
    */ @JsMethod - public void respond(@JsNullable Object content) { + public void respond(@JsNullable ResourceContentUnion content) { // Build RemoteFileSourceMetaResponse proto RemoteFileSourceMetaResponse response = new RemoteFileSourceMetaResponse(); @@ -352,12 +332,12 @@ public void respond(@JsNullable Object content) { } else { response.setFound(true); - // Convert content to bytes - if (content instanceof String) { + // Convert content to bytes using union type methods + if (content.isString()) { TextEncoder textEncoder = new TextEncoder(); - response.setContent(textEncoder.encode((String) content)); - } else if (content instanceof Uint8Array) { - response.setContent((Uint8Array) content); + response.setContent(textEncoder.encode(content.asString())); + } else if (content.isUint8Array()) { + response.setContent(content.asUint8Array()); } else { throw new IllegalArgumentException("Content must be a String, Uint8Array, or null"); } diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java new file mode 100644 index 00000000000..26135d2f1ba --- /dev/null +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java @@ -0,0 +1,49 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// +package io.deephaven.web.client.api.remotefilesource; + +import com.vertispan.tsdefs.annotations.TsUnion; +import com.vertispan.tsdefs.annotations.TsUnionMember; +import elemental2.core.Uint8Array; +import jsinterop.annotations.JsOverlay; +import jsinterop.annotations.JsPackage; +import jsinterop.annotations.JsType; +import jsinterop.base.Js; + +/** + * Union type for resource content that can be either a String or Uint8Array. + */ +@TsUnion +@JsType(name = "?", namespace = JsPackage.GLOBAL, isNative = true) +public interface ResourceContentUnion { + @JsOverlay + static ResourceContentUnion of(Object o) { + return Js.cast(o); + } + + @JsOverlay + default boolean isString() { + // Cast to (Object) since Java only "knows" that `this` is `ResourceContentUnion` type which cannot have a + // subclass that is also a String. + return (Object) this instanceof String; + } + + @JsOverlay + default boolean isUint8Array() { + return this instanceof Uint8Array; + } + + @TsUnionMember + @JsOverlay + default String asString() { + return Js.cast(this); + } + + @TsUnionMember + @JsOverlay + default Uint8Array asUint8Array() { + return (Uint8Array) this; + } +} + From f9beca5df348ca44db642ca72291572b01ec3a48 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 13 Jan 2026 17:13:45 -0600 Subject: [PATCH 53/78] Changed to getResource (#DH-20578) --- .../deephaven/engine/util/RemoteFileSourceClassLoader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 470b6c88767..c4b3601a919 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -68,7 +68,7 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { } /** - * Finds the resource with the specified name by checking registered providers. + * Gets the resource with the specified name by checking registered providers. * *

    This method iterates through all registered providers to see if any can source the requested resource. * If a provider can handle the resource, a custom URL with protocol "remotefile://" is returned. @@ -78,7 +78,7 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { * @return a URL for reading the resource, or null if the resource could not be found */ @Override - protected URL findResource(String name) { + public URL getResource(String name) { RemoteFileSourceProvider provider = null; for (RemoteFileSourceProvider candidate : providers) { if (candidate.isActive() && candidate.canSourceResource(name)) { @@ -95,7 +95,7 @@ protected URL findResource(String name) { } } - return super.findResource(name); + return super.getResource(name); } /** From 81a9de25af25455d6dbddc3a197b8c24c6065993 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 13 Jan 2026 18:12:06 -0600 Subject: [PATCH 54/78] Addressed review comments (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 25 +++++++++++++++++-- .../engine/util/GroovyDeephavenSession.java | 2 +- .../RemoteFileSourceMessageStream.java | 16 +++++++++--- .../JsRemoteFileSourceService.java | 22 +++------------- 4 files changed, 41 insertions(+), 24 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index c4b3601a919..9a28f6323c0 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -35,9 +35,27 @@ public class RemoteFileSourceClassLoader extends ClassLoader { * * @param parent the parent class loader for delegation */ - public RemoteFileSourceClassLoader(ClassLoader parent) { + private RemoteFileSourceClassLoader(ClassLoader parent) { super(parent); - instance = this; + } + + /** + * Initializes the singleton RemoteFileSourceClassLoader instance with the specified parent class loader. + * + *

    This method must be called exactly once before any calls to {@link #getInstance()}. The method is + * synchronized to prevent race conditions when multiple threads attempt initialization. + * + * @param parent the parent class loader for delegation + * @return the newly created singleton instance + * @throws IllegalStateException if the instance has already been initialized + */ + public static synchronized RemoteFileSourceClassLoader initialize(ClassLoader parent) { + if (instance != null) { + throw new IllegalStateException("RemoteFileSourceClassLoader is already initialized"); + } + + instance = new RemoteFileSourceClassLoader(parent); + return instance; } /** @@ -46,6 +64,9 @@ public RemoteFileSourceClassLoader(ClassLoader parent) { * @return the singleton instance, or null if not yet initialized */ public static RemoteFileSourceClassLoader getInstance() { + if (instance == null) { + throw new IllegalStateException("RemoteFileSourceClassLoader is not yet initialized"); + } return instance; } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index ebec4bc2977..9d9154317f4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -96,7 +96,7 @@ public class GroovyDeephavenSession extends AbstractScriptSession mapping = new ConcurrentHashMap<>(); @Override diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index cbafb2e1dc4..a50316aa75c 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -159,10 +159,20 @@ public boolean isActive() { /** * Sets the execution context with the active message stream and resource paths. - * This should be called when a script execution begins. * - * @param messageStream the message stream to set as active - * @param resourcePaths list of resource paths to resolve from remote source + *

    This static method establishes which message stream instance should be considered "active" for + * resource requests, and which resource paths should be resolved from that remote source. Only one + * execution context can be active at a time across all instances. + * + *

    In multi-client scenarios (Community Core), this ensures that only the + * message stream for the currently executing script is active, preventing resource requests from + * being serviced by the wrong client connection. + * + *

    Typical Usage: Called at the beginning of script execution to establish which .groovy + * files should be sourced from the remote client rather than the local classpath. + * + * @param messageStream the message stream to set as active (must not be null) + * @param resourcePaths list of resource paths (e.g., "package/MyScript.groovy") to resolve from remote source * @throws IllegalArgumentException if messageStream is null */ public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, List resourcePaths) { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 957ee628c4b..3de111d1574 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -47,16 +47,11 @@ *

    * Events: *

      - *
    • {@link #EVENT_MESSAGE}: Fired for unrecognized messages from the server
      • *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client
      • *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") public class JsRemoteFileSourceService extends HasEventHandling { - /** Event name for generic messages from the server */ - @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") - public static final String EVENT_MESSAGE = "message"; - /** Event name for resource request events from the server */ @JsProperty(namespace = "dh.remotefilesource.RemoteFileSourceService") public static final String EVENT_REQUEST_SOURCE = "requestsource"; @@ -155,7 +150,7 @@ public static Promise fetchPlugin(WorkerConnection co * @return a promise that resolves to this service instance when the connection is established */ private Promise connect() { - widget.addEventListener("message", this::handleMessage); + widget.addEventListener(JsWidget.EVENT_MESSAGE, this::handleMessage); return widget.refetch().then(w -> Promise.resolve(this)); } @@ -171,9 +166,8 @@ private void handleMessage(Event event) { try { message = RemoteFileSourceServerRequest.deserializeBinary(payload); } catch (Exception e) { - // Failed to parse as proto, fire generic message event - handleUnknownMessage(event); - return; + // Failed to parse as proto + throw new IllegalStateException("Received unparseable message from server", e); } // Route the parsed message to the appropriate handler @@ -182,7 +176,7 @@ private void handleMessage(Event event) { } else if (message.hasSetExecutionContextResponse()) { handleSetExecutionContextResponse(message); } else { - handleUnknownMessage(event); + throw new IllegalStateException("Received unknown message type from server"); } } @@ -211,14 +205,6 @@ private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest mes } } - /** - * Handles an unknown or unparseable message from the server. - * - * @param event the message event - */ - private void handleUnknownMessage(Event event) { - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_MESSAGE, event.getDetail()), 0); - } /** * Sets the execution context on the server to identify this message stream as active From 8a9481e0f17fc654495d8ff893e644fee8a15db0 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 13 Jan 2026 18:18:44 -0600 Subject: [PATCH 55/78] Addressed review comments (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 5 ++++- .../RemoteFileSourceMessageStream.java | 19 ++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 9a28f6323c0..7e751a45998 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -61,7 +61,10 @@ public static synchronized RemoteFileSourceClassLoader initialize(ClassLoader pa /** * Returns the singleton instance of the RemoteFileSourceClassLoader. * - * @return the singleton instance, or null if not yet initialized + *

    This method requires that {@link #initialize(ClassLoader)} has been called first. + * + * @return the singleton instance + * @throws IllegalStateException if the instance has not yet been initialized via {@link #initialize(ClassLoader)} */ public static RemoteFileSourceClassLoader getInstance() { if (instance == null) { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index a50316aa75c..f09bf2a4c1b 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -216,7 +216,8 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun } else if (message.hasSetExecutionContext()) { handleSetExecutionContext(message.getRequestId(), message.getSetExecutionContext().getResourcePathsList()); } else { - log.warn().append("Received unknown message type from client").endl(); + log.error().append("Received unknown message type from client").endl(); + throw new ObjectCommunicationException("Received unknown message type from client"); } } catch (InvalidProtocolBufferException e) { log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); @@ -311,13 +312,8 @@ public void onClose() { */ private void registerWithClassLoader() { RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); - - if (classLoader != null) { - classLoader.registerProvider(this); - log.info().append("Registered RemoteFileSourceMessageStream provider with RemoteFileSourceClassLoader").endl(); - } else { - log.warn().append("RemoteFileSourceClassLoader not available").endl(); - } + classLoader.registerProvider(this); + log.info().append("Registered RemoteFileSourceMessageStream provider with RemoteFileSourceClassLoader").endl(); } /** @@ -325,11 +321,8 @@ private void registerWithClassLoader() { */ private void unregisterFromClassLoader() { RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); - - if (classLoader != null) { - classLoader.unregisterProvider(this); - log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); - } + classLoader.unregisterProvider(this); + log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); } From 2a293e92a529c62158025420abb28b4906824bae Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 14 Jan 2026 12:17:32 -0600 Subject: [PATCH 56/78] Addressed review comments (#DH-20578) --- .../deephaven/web/client/api/CoreClient.java | 7 ++++- .../JsRemoteFileSourceService.java | 29 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java index 78ba535a24f..1eb0247b1f8 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/CoreClient.java @@ -47,6 +47,7 @@ public class CoreClient extends HasEventHandling { LOGIN_TYPE_ANONYMOUS = "anonymous"; private final IdeConnection ideConnection; + private Promise remoteFileSourceServicePromise; public CoreClient(String serverUrl, @TsTypeRef(ConnectOptions.class) @JsOptional Object connectOptions) { ideConnection = new IdeConnection(serverUrl, connectOptions); @@ -143,7 +144,11 @@ public Promise onConnected(@JsOptional Double timeoutInMillis) { } public Promise getRemoteFileSourceService() { - return JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + if (remoteFileSourceServicePromise == null) { + remoteFileSourceServicePromise = JsRemoteFileSourceService.fetchPlugin(ideConnection.connection.get()); + } + + return remoteFileSourceServicePromise; } public Promise getServerConfigValues() { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 3de111d1574..5ca57e60266 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -23,9 +23,11 @@ import io.deephaven.web.client.api.Callbacks; import io.deephaven.web.client.api.JsProtobufUtils; import io.deephaven.web.client.api.event.Event; +import io.deephaven.web.client.api.event.EventFn; import io.deephaven.web.client.api.WorkerConnection; import io.deephaven.web.client.api.event.HasEventHandling; import io.deephaven.web.client.api.widget.JsWidget; +import io.deephaven.web.shared.fu.RemoverFn; import io.deephaven.web.client.api.widget.WidgetMessageDetails; import io.deephaven.web.client.fu.LazyPromise; import jsinterop.annotations.JsIgnore; @@ -47,7 +49,10 @@ *

    * Events: *

      - *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client
      • + *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client. + * This event MUST have exactly one listener registered. Attempting to register more than one listener + * will throw an IllegalStateException. Receiving a resource request without a registered listener + * will also throw an IllegalStateException.
      • *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") @@ -72,6 +77,23 @@ private JsRemoteFileSourceService(JsWidget widget) { this.widget = widget; } + /** + * Overrides addEventListener to enforce that EVENT_REQUEST_SOURCE can only have one listener. + * + * @param name the name of the event to listen for + * @param callback a function to call when the event occurs + * @return Returns a cleanup function. + * @param the type of the data that the event will provide + */ + @Override + public RemoverFn addEventListener(String name, EventFn callback) { + if (EVENT_REQUEST_SOURCE.equals(name) && hasListeners(EVENT_REQUEST_SOURCE)) { + throw new IllegalStateException( + "EVENT_REQUEST_SOURCE already has a listener. Only one listener is allowed for this event."); + } + return super.addEventListener(name, callback); + } + /** * Fetches the FlightInfo for the plugin fetch command. * @@ -186,6 +208,11 @@ private void handleMessage(Event event) { * @param message the server request message */ private void handleMetaRequest(RemoteFileSourceServerRequest message) { + if (!hasListeners(EVENT_REQUEST_SOURCE)) { + throw new IllegalStateException( + "Received resource request from server but no listener is registered for EVENT_REQUEST_SOURCE. " + + "A listener must be registered to handle resource requests."); + } RemoteFileSourceMetaRequest request = message.getMetaRequest(); DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, new ResourceRequestEvent(message.getRequestId(), request)), 0); From 1f000f3cb7385760ada804eec9418948c6d43d3a Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 14 Jan 2026 12:28:50 -0600 Subject: [PATCH 57/78] Removed timeout (#DH-20578) --- .../client/api/remotefilesource/JsRemoteFileSourceService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 5ca57e60266..b706d895467 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -214,8 +214,7 @@ private void handleMetaRequest(RemoteFileSourceServerRequest message) { + "A listener must be registered to handle resource requests."); } RemoteFileSourceMetaRequest request = message.getMetaRequest(); - DomGlobal.setTimeout(ignore -> fireEvent(EVENT_REQUEST_SOURCE, - new ResourceRequestEvent(message.getRequestId(), request)), 0); + fireEvent(EVENT_REQUEST_SOURCE, new ResourceRequestEvent(message.getRequestId(), request)); } /** From 3f3ed19a8c977d1d45f33524d9852596cda12ce9 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 19 Feb 2026 18:01:17 -0600 Subject: [PATCH 58/78] Attempt at clearing class cache (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 44 +++++ .../engine/util/AbstractScriptSession.java | 22 ++- .../engine/util/GroovyDeephavenSession.java | 161 +++++++++++++++++- 3 files changed, 222 insertions(+), 5 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 7e751a45998..0221e88ce7d 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -30,6 +30,10 @@ public class RemoteFileSourceClassLoader extends ClassLoader { private static volatile RemoteFileSourceClassLoader instance; private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); + // Track which remote resources have been fetched (for cache invalidation optimization) + // Using a thread-safe Set to ensure uniqueness and O(1) contains() performance + private final java.util.Set fetchedRemoteResources = java.util.Collections.synchronizedSet(new java.util.HashSet<>()); + /** * Constructs a new RemoteFileSourceClassLoader with the specified parent class loader. * @@ -91,6 +95,44 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { providers.remove(provider); } + /** + * Records that a remote resource has been fetched. + * This is used to track whether cache invalidation is needed. + * + * @param resourceName the name of the resource that was fetched + */ + void recordRemoteResourceFetch(String resourceName) { + fetchedRemoteResources.add(resourceName); + } + + /** + * Returns whether any remote resources have been fetched. + * This can be used to determine if cache invalidation is necessary. + * + * @return true if any remote resources have been fetched, false otherwise + */ + public boolean hasRemoteResourcesBeenFetched() { + return !fetchedRemoteResources.isEmpty(); + } + + /** + * Returns a copy of the list of remote resources that have been fetched. + * This can be used for selective cache invalidation. + * + * @return list of fetched remote resource names + */ + public java.util.List getFetchedRemoteResources() { + return new java.util.ArrayList<>(fetchedRemoteResources); + } + + /** + * Clears the tracking of fetched remote resources. + * This should be called after cache invalidation to reset the state. + */ + public void clearFetchedResourcesTracking() { + fetchedRemoteResources.clear(); + } + /** * Gets the resource with the specified name by checking registered providers. * @@ -189,6 +231,8 @@ public void connect() throws IOException { .orTimeout(RESOURCE_TIMEOUT_SECONDS, TimeUnit.SECONDS) .get(); connected = true; + // Track that this remote resource was fetched + RemoteFileSourceClassLoader.getInstance().recordRemoteResourceFetch(resourceName); } catch (Exception e) { throw new IOException("Failed to fetch remote resource: " + resourceName, e); } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index 76f75200757..d3d3e220b39 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -69,7 +69,8 @@ private static void ensureDirectory(final Path directory) { protected final File classCacheDirectory; private final ScriptSessionQueryScope queryScope; - protected final ExecutionContext executionContext; + // DH-20578: Not final so it can be updated with fresh QueryCompiler when remote sources are detected + protected ExecutionContext executionContext; private S lastSnapshot; @@ -95,9 +96,26 @@ protected AbstractScriptSession( this.classCacheDirectory = classCacheDirectory; queryScope = new ScriptSessionQueryScope(); + + executionContext = createExecutionContext(updateGraph, operationInitializer, parentClassLoader); + } + + /** + * DH-20578: Creates an ExecutionContext with a QueryCompiler based on the provided ClassLoader. + * This allows updating the ExecutionContext when fresh ClassLoaders are needed (e.g., for remote file sources). + * + * @param updateGraph the update graph for the context + * @param operationInitializer the operation initializer for the context + * @param parentClassLoader the ClassLoader to use for creating the QueryCompiler + * @return a new ExecutionContext with a QueryCompiler based on the provided ClassLoader + */ + protected ExecutionContext createExecutionContext( + final UpdateGraph updateGraph, + final OperationInitializer operationInitializer, + final ClassLoader parentClassLoader) { final QueryCompiler compilerContext = QueryCompilerImpl.create(classCacheDirectory, parentClassLoader); - executionContext = ExecutionContext.newBuilder() + return ExecutionContext.newBuilder() .markSystemic() .newQueryLibrary() .setQueryScope(queryScope) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 9d9154317f4..740de862f0d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -162,7 +162,7 @@ public SafeCloseable setScriptPrefix(final String newPrefix) { } } - private final DeephavenGroovyShell groovyShell; + private DeephavenGroovyShell groovyShell; public static GroovyDeephavenSession of( final UpdateGraph updateGraph, @@ -343,8 +343,9 @@ protected void evaluate(String command, String scriptName) { final String currentScriptName = scriptName == null ? DEFAULT_SCRIPT_PREFIX : scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); - try (final SafeCloseable ignored = groovyShell.setScriptPrefix(currentScriptName)) { + // Execute the script + try (final SafeCloseable ignored = groovyShell.setScriptPrefix(currentScriptName)) { updateClassloader(lastCommand); try { @@ -356,6 +357,40 @@ protected void evaluate(String command, String scriptName) { } catch (Exception e) { throw wrapAndRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix); } + } finally { + // DH-20578: Handle cache invalidation AFTER execution + final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); + final boolean hasRemoteSources = remoteLoader.hasRemoteResourcesBeenFetched(); + + if (hasRemoteSources) { + log.debug("DH-20578: Remote sources detected - cleaning up and preparing for next execution"); + + // Get list of remote resources that were used + final java.util.List remoteResources = remoteLoader.getFetchedRemoteResources(); + + // Delete .class files for remote sources so next execution doesn't load stale cached versions + if (remoteResources != null && classCacheDirectory != null && classCacheDirectory.exists()) { + deleteClassFilesForRemoteSources(classCacheDirectory, remoteResources); + } + + // Create fresh GroovyClassLoader for next execution + CompilerConfiguration freshConfig = new CompilerConfiguration(); + freshConfig.getCompilationCustomizers().add(consoleImports); + freshConfig.setTargetDirectory(classCacheDirectory); + GroovyClassLoader freshClassLoader = new GroovyClassLoader(STATIC_LOADER, freshConfig); + + // Update the session's ExecutionContext with fresh QueryCompiler + executionContext = createExecutionContext( + executionContext.getUpdateGraph(), + executionContext.getOperationInitializer(), + freshClassLoader); + + // Permanently replace groovyShell with fresh shell for next execution + groovyShell = new DeephavenGroovyShell(freshClassLoader, groovyShell.getContext(), freshConfig); + + // Clear tracking after setup for next execution + remoteLoader.clearFetchedResourcesTracking(); + } } } @@ -689,7 +724,7 @@ && isAnInteger(aClass.getName().substring(SCRIPT_PREFIX.length()))) { // only increment QueryLibrary version if some dynamic class overrides an existing class if (!dynamicClasses.add(entry.getKey()) && !notifiedQueryLibrary) { notifiedQueryLibrary = true; - executionContext.getQueryLibrary().updateVersionString(); + getExecutionContext().getQueryLibrary().updateVersionString(); } try { @@ -717,6 +752,126 @@ private static boolean isAnInteger(final String s) { } } + /** + * DH-20578: Delete .class files from the cache directory to force recompilation. + * This is necessary because GroovyClassLoader can reload .class files from disk even after clearCache(). + */ + private static void deleteClassFiles(File directory) { + if (directory == null || !directory.exists() || !directory.isDirectory()) { + return; + } + + File[] files = directory.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + if (file.isDirectory()) { + deleteClassFiles(file); + } else if (file.getName().endsWith(".class")) { + if (!file.delete()) { + log.warn("DH-20578: Failed to delete cached class file: " + file.getAbsolutePath()); + } + } + } + } + + /** + * DH-20578: Selectively delete .class files only for classes that correspond to remote sources. + * This is more efficient than deleting all .class files. + * + * @param directory the directory to search for .class files + * @param remoteResources list of remote resource names (e.g., "test2/notebook/MyNotebook.groovy") + */ + private static void deleteClassFilesForRemoteSources(File directory, java.util.List remoteResources) { + if (directory == null || !directory.exists() || !directory.isDirectory() || remoteResources.isEmpty()) { + return; + } + + // Convert remote resource names to class file paths + // e.g., "test2/notebook/MyNotebook.groovy" -> "test2/notebook/MyNotebook.class" + java.util.Set classFilePaths = new java.util.HashSet<>(); + for (String resource : remoteResources) { + if (resource.endsWith(".groovy")) { + String classPath = resource.substring(0, resource.length() - 7) + ".class"; // Remove .groovy, add .class + classFilePaths.add(classPath); + log.debug("DH-20578: Will delete .class file for remote source: " + classPath); + } + } + + deleteMatchingClassFiles(directory, classFilePaths); + } + + /** + * Recursively delete .class files that match the given set of paths. + */ + private static void deleteMatchingClassFiles(File directory, java.util.Set classFilePaths) { + if (directory == null || !directory.exists() || !directory.isDirectory()) { + return; + } + + File[] files = directory.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + if (file.isDirectory()) { + deleteMatchingClassFiles(file, classFilePaths); + } else if (file.getName().endsWith(".class")) { + // Get relative path from classCacheDirectory + String relativePath = getRelativePath(file); + if (classFilePaths.contains(relativePath)) { + if (file.delete()) { + log.info("DH-20578: Deleted cached .class file for remote source: " + relativePath); + } else { + log.warn("DH-20578: Failed to delete cached class file: " + file.getAbsolutePath()); + } + } + } + } + } + + /** + * Get a relative path string from a file by walking up to find a reasonable base. + * This is a simple implementation that uses forward slashes. + */ + private static String getRelativePath(File file) { + // Simple approach: get the path and look for package-like structure + String path = file.getPath(); + // Replace backslashes with forward slashes for consistency + path = path.replace('\\', '/'); + + // Try to find a reasonable starting point by looking for common package prefixes + // For now, just return the path segments that look like packages + int lastSeparator = path.lastIndexOf('/'); + if (lastSeparator > 0) { + // Look backwards to find what looks like a package structure + // This is a simple heuristic - could be improved + String[] segments = path.split("/"); + StringBuilder result = new StringBuilder(); + boolean inPackage = false; + for (int i = 0; i < segments.length; i++) { + String segment = segments[i]; + // Start capturing when we see a lowercase segment (likely package name) + if (!inPackage && segment.length() > 0 && Character.isLowerCase(segment.charAt(0))) { + inPackage = true; + } + if (inPackage) { + if (result.length() > 0) { + result.append('/'); + } + result.append(segment); + } + } + if (result.length() > 0) { + return result.toString(); + } + } + return file.getName(); + } + @Override protected GroovySnapshot emptySnapshot() { return new GroovySnapshot(Collections.emptyMap()); From f08fd835648730389f359ff509f3ea27ae6d5aef Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 20 Feb 2026 15:04:51 -0600 Subject: [PATCH 59/78] Fixed bug with cache clearing only after running script 2x (#DH-20578) --- .../engine/util/RemoteFileSourceClassLoader.java | 15 +++++++++++++++ .../engine/util/GroovyDeephavenSession.java | 5 ++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 0221e88ce7d..d77a7761740 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -133,6 +133,21 @@ public void clearFetchedResourcesTracking() { fetchedRemoteResources.clear(); } + /** + * Returns whether there are any active providers with resource paths configured. + * This indicates that remote sources are available and may be used. + * + * @return true if any provider is active and has resources it can source, false otherwise + */ + public boolean hasActiveProviders() { + for (RemoteFileSourceProvider candidate : providers) { + if (candidate.isActive()) { + return true; + } + } + return false; + } + /** * Gets the resource with the specified name by checking registered providers. * diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 740de862f0d..9e774d7dd7f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -360,7 +360,10 @@ protected void evaluate(String command, String scriptName) { } finally { // DH-20578: Handle cache invalidation AFTER execution final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); - final boolean hasRemoteSources = remoteLoader.hasRemoteResourcesBeenFetched(); + // Check both: were resources fetched in this execution? OR is there an active provider now? + // The second check handles the case where remote config was just added but resources not yet fetched + final boolean hasRemoteSources = remoteLoader.hasRemoteResourcesBeenFetched() + || remoteLoader.hasActiveProviders(); if (hasRemoteSources) { log.debug("DH-20578: Remote sources detected - cleaning up and preparing for next execution"); From 75ff3db5afeefeb1b5fcd3d0632a7fdee7c863d8 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 23 Feb 2026 18:17:21 -0600 Subject: [PATCH 60/78] Cleaned up some implementation of groovy session clearing (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 35 +++-- .../engine/util/RemoteFileSourceProvider.java | 8 + .../engine/util/GroovyDeephavenSession.java | 147 ++++++++++-------- .../RemoteFileSourceMessageStream.java | 14 ++ 4 files changed, 131 insertions(+), 73 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index d77a7761740..5959eca23c4 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -10,6 +10,11 @@ import java.net.URL; import java.net.URLConnection; import java.net.URLStreamHandler; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; @@ -32,7 +37,7 @@ public class RemoteFileSourceClassLoader extends ClassLoader { // Track which remote resources have been fetched (for cache invalidation optimization) // Using a thread-safe Set to ensure uniqueness and O(1) contains() performance - private final java.util.Set fetchedRemoteResources = java.util.Collections.synchronizedSet(new java.util.HashSet<>()); + private final Set fetchedRemoteResources = Collections.synchronizedSet(new HashSet<>()); /** * Constructs a new RemoteFileSourceClassLoader with the specified parent class loader. @@ -105,15 +110,6 @@ void recordRemoteResourceFetch(String resourceName) { fetchedRemoteResources.add(resourceName); } - /** - * Returns whether any remote resources have been fetched. - * This can be used to determine if cache invalidation is necessary. - * - * @return true if any remote resources have been fetched, false otherwise - */ - public boolean hasRemoteResourcesBeenFetched() { - return !fetchedRemoteResources.isEmpty(); - } /** * Returns a copy of the list of remote resources that have been fetched. @@ -121,8 +117,8 @@ public boolean hasRemoteResourcesBeenFetched() { * * @return list of fetched remote resource names */ - public java.util.List getFetchedRemoteResources() { - return new java.util.ArrayList<>(fetchedRemoteResources); + public List getFetchedRemoteResources() { + return new ArrayList<>(fetchedRemoteResources); } /** @@ -148,6 +144,21 @@ public boolean hasActiveProviders() { return false; } + /** + * Returns whether there are any active providers with non-empty resource paths configured. + * This indicates that remote sources are actually configured, not just that the execution context is set. + * + * @return true if any provider is active and has resource paths configured, false otherwise + */ + public boolean hasConfiguredRemoteSources() { + for (RemoteFileSourceProvider candidate : providers) { + if (candidate.isActive() && candidate.hasConfiguredResources()) { + return true; + } + } + return false; + } + /** * Gets the resource with the specified name by checking registered providers. * diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index 377172ea605..08fe4f9240b 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -26,6 +26,14 @@ public interface RemoteFileSourceProvider { */ boolean isActive(); + /** + * Check if this provider has any resource paths configured. + * A provider can be active (execution context set) but have no resource paths configured. + * + * @return true if this provider has resource paths configured, false otherwise + */ + boolean hasConfiguredResources(); + /** * Request a resource from the remote source. * diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 9e774d7dd7f..6580e880e60 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -164,6 +164,10 @@ public SafeCloseable setScriptPrefix(final String newPrefix) { private DeephavenGroovyShell groovyShell; + // Track whether remote sources were active in the previous execution + // Used to detect transitions from no remote sources to active remote sources + private boolean previousEvalHadRemoteSources = false; + public static GroovyDeephavenSession of( final UpdateGraph updateGraph, final OperationInitializer operationInitializer, @@ -332,6 +336,51 @@ protected T getVariable(String name) { } } + /** + * Clears cached .class files and refreshes the GroovyClassLoader and associated shell. + * This ensures that classes will be recompiled from source on the next execution. + * + * @param classPathsToClear set of specific class file paths to delete (e.g., "test2/notebook/MyNotebook.class"). + * If null, all .class files in the cache directory will be deleted. + * If empty, no files will be deleted and this method does nothing. + * @throws IllegalStateException if the cache directory does not exist + */ + private void refreshClassLoader(@Nullable Set classPathsToClear) { + // If no cache directory exists, this is an error - we shouldn't be trying to refresh without a cache + if (classCacheDirectory == null || !classCacheDirectory.exists()) { + throw new IllegalStateException("Cannot refresh classloader: cache directory does not exist"); + } + + // If empty set, nothing to clear - skip entirely + if (classPathsToClear != null && classPathsToClear.isEmpty()) { + return; + } + + // Delete cached .class files + if (classPathsToClear == null) { + // Clear all cached .class files + deleteClassFiles(classCacheDirectory); + } else { + // Selectively clear only specified .class files + deleteMatchingClassFiles(classCacheDirectory, classPathsToClear); + } + + // Create a fresh GroovyClassLoader + CompilerConfiguration freshConfig = new CompilerConfiguration(); + freshConfig.getCompilationCustomizers().add(consoleImports); + freshConfig.setTargetDirectory(classCacheDirectory); + GroovyClassLoader freshClassLoader = new GroovyClassLoader(STATIC_LOADER, freshConfig); + + // Update the session's ExecutionContext with fresh QueryCompiler + executionContext = createExecutionContext( + executionContext.getUpdateGraph(), + executionContext.getOperationInitializer(), + freshClassLoader); + + // Permanently replace groovyShell with fresh shell + groovyShell = new DeephavenGroovyShell(freshClassLoader, groovyShell.getContext(), freshConfig); + } + @Override protected void evaluate(String command, String scriptName) { grepScriptImports(removeComments(command)); @@ -344,6 +393,19 @@ protected void evaluate(String command, String scriptName) { ? DEFAULT_SCRIPT_PREFIX : scriptName.replaceAll("[^0-9A-Za-z_]", "_").replaceAll("(^[0-9])", "_$1"); + final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); + final boolean hasRemoteSources = remoteLoader.hasConfiguredRemoteSources(); + + // If we are switching from no remote sources to active remote sources, we don't know which classes in the + // cache will be sourced remotely, so we clear the entire cache before execution. + if (!previousEvalHadRemoteSources && hasRemoteSources) { + log.debug("Remote file sourcing enabled. Clearing class cache."); + refreshClassLoader(null); // null = clear all cache files + } + + // Update state tracker for next execution + previousEvalHadRemoteSources = hasRemoteSources; + // Execute the script try (final SafeCloseable ignored = groovyShell.setScriptPrefix(currentScriptName)) { updateClassloader(lastCommand); @@ -358,40 +420,28 @@ protected void evaluate(String command, String scriptName) { throw wrapAndRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix); } } finally { - // DH-20578: Handle cache invalidation AFTER execution - final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); - // Check both: were resources fetched in this execution? OR is there an active provider now? - // The second check handles the case where remote config was just added but resources not yet fetched - final boolean hasRemoteSources = remoteLoader.hasRemoteResourcesBeenFetched() - || remoteLoader.hasActiveProviders(); - - if (hasRemoteSources) { - log.debug("DH-20578: Remote sources detected - cleaning up and preparing for next execution"); - - // Get list of remote resources that were used - final java.util.List remoteResources = remoteLoader.getFetchedRemoteResources(); - - // Delete .class files for remote sources so next execution doesn't load stale cached versions - if (remoteResources != null && classCacheDirectory != null && classCacheDirectory.exists()) { - deleteClassFilesForRemoteSources(classCacheDirectory, remoteResources); + // Get list of remote resources that were used in this execution + final List remoteResources = remoteLoader.getFetchedRemoteResources(); + + // Scenario 2: Remote sources were fetched during execution + // Selectively clear those specific .class files and refresh classloader for next execution + if (!remoteResources.isEmpty()) { + log.debug("Remote resources were fetched - selectively clearing cache after execution"); + + // Convert remote resource names to class file paths + // e.g., "test2/notebook/MyNotebook.groovy" -> "test2/notebook/MyNotebook.class" + Set classFilePaths = new HashSet<>(); + for (String resource : remoteResources) { + if (resource.endsWith(".groovy")) { + String classPath = resource.substring(0, resource.length() - 7) + ".class"; + classFilePaths.add(classPath); + log.debug("Will delete .class file for remote source: " + classPath); + } } - // Create fresh GroovyClassLoader for next execution - CompilerConfiguration freshConfig = new CompilerConfiguration(); - freshConfig.getCompilationCustomizers().add(consoleImports); - freshConfig.setTargetDirectory(classCacheDirectory); - GroovyClassLoader freshClassLoader = new GroovyClassLoader(STATIC_LOADER, freshConfig); + refreshClassLoader(classFilePaths); - // Update the session's ExecutionContext with fresh QueryCompiler - executionContext = createExecutionContext( - executionContext.getUpdateGraph(), - executionContext.getOperationInitializer(), - freshClassLoader); - - // Permanently replace groovyShell with fresh shell for next execution - groovyShell = new DeephavenGroovyShell(freshClassLoader, groovyShell.getContext(), freshConfig); - - // Clear tracking after setup for next execution + // Always clear tracking after selective cleanup to prevent accumulation remoteLoader.clearFetchedResourcesTracking(); } } @@ -756,7 +806,7 @@ private static boolean isAnInteger(final String s) { } /** - * DH-20578: Delete .class files from the cache directory to force recompilation. + * Delete .class files from the cache directory to force recompilation. * This is necessary because GroovyClassLoader can reload .class files from disk even after clearCache(). */ private static void deleteClassFiles(File directory) { @@ -774,42 +824,17 @@ private static void deleteClassFiles(File directory) { deleteClassFiles(file); } else if (file.getName().endsWith(".class")) { if (!file.delete()) { - log.warn("DH-20578: Failed to delete cached class file: " + file.getAbsolutePath()); + log.warn("Failed to delete cached class file: " + file.getAbsolutePath()); } } } } - /** - * DH-20578: Selectively delete .class files only for classes that correspond to remote sources. - * This is more efficient than deleting all .class files. - * - * @param directory the directory to search for .class files - * @param remoteResources list of remote resource names (e.g., "test2/notebook/MyNotebook.groovy") - */ - private static void deleteClassFilesForRemoteSources(File directory, java.util.List remoteResources) { - if (directory == null || !directory.exists() || !directory.isDirectory() || remoteResources.isEmpty()) { - return; - } - - // Convert remote resource names to class file paths - // e.g., "test2/notebook/MyNotebook.groovy" -> "test2/notebook/MyNotebook.class" - java.util.Set classFilePaths = new java.util.HashSet<>(); - for (String resource : remoteResources) { - if (resource.endsWith(".groovy")) { - String classPath = resource.substring(0, resource.length() - 7) + ".class"; // Remove .groovy, add .class - classFilePaths.add(classPath); - log.debug("DH-20578: Will delete .class file for remote source: " + classPath); - } - } - - deleteMatchingClassFiles(directory, classFilePaths); - } /** * Recursively delete .class files that match the given set of paths. */ - private static void deleteMatchingClassFiles(File directory, java.util.Set classFilePaths) { + private static void deleteMatchingClassFiles(File directory, Set classFilePaths) { if (directory == null || !directory.exists() || !directory.isDirectory()) { return; } @@ -827,9 +852,9 @@ private static void deleteMatchingClassFiles(File directory, java.util.Set Date: Tue, 24 Feb 2026 09:48:41 -0600 Subject: [PATCH 61/78] Clean up (#DH-20578) --- .../engine/util/GroovyDeephavenSession.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 6580e880e60..2ed59b1cb7a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -164,8 +164,6 @@ public SafeCloseable setScriptPrefix(final String newPrefix) { private DeephavenGroovyShell groovyShell; - // Track whether remote sources were active in the previous execution - // Used to detect transitions from no remote sources to active remote sources private boolean previousEvalHadRemoteSources = false; public static GroovyDeephavenSession of( @@ -342,8 +340,9 @@ protected T getVariable(String name) { * * @param classPathsToClear set of specific class file paths to delete (e.g., "test2/notebook/MyNotebook.class"). * If null, all .class files in the cache directory will be deleted. - * If empty, no files will be deleted and this method does nothing. + * Must not be empty. * @throws IllegalStateException if the cache directory does not exist + * @throws IllegalArgumentException if classPathsToClear is empty */ private void refreshClassLoader(@Nullable Set classPathsToClear) { // If no cache directory exists, this is an error - we shouldn't be trying to refresh without a cache @@ -351,16 +350,14 @@ private void refreshClassLoader(@Nullable Set classPathsToClear) { throw new IllegalStateException("Cannot refresh classloader: cache directory does not exist"); } - // If empty set, nothing to clear - skip entirely - if (classPathsToClear != null && classPathsToClear.isEmpty()) { - return; - } - - // Delete cached .class files if (classPathsToClear == null) { // Clear all cached .class files deleteClassFiles(classCacheDirectory); - } else { + } + else if(classPathsToClear.isEmpty()) { + throw new IllegalArgumentException("classPathsToClear cannot be empty"); + } + else { // Selectively clear only specified .class files deleteMatchingClassFiles(classCacheDirectory, classPathsToClear); } From 81785f3e27f5aa811cf4946da50487f63f3a0e0b Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 24 Feb 2026 10:06:56 -0600 Subject: [PATCH 62/78] Clean up (#DH-20578) --- .../engine/util/GroovyDeephavenSession.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 2ed59b1cb7a..7240f700425 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -363,19 +363,19 @@ else if(classPathsToClear.isEmpty()) { } // Create a fresh GroovyClassLoader - CompilerConfiguration freshConfig = new CompilerConfiguration(); - freshConfig.getCompilationCustomizers().add(consoleImports); - freshConfig.setTargetDirectory(classCacheDirectory); - GroovyClassLoader freshClassLoader = new GroovyClassLoader(STATIC_LOADER, freshConfig); + CompilerConfiguration config = new CompilerConfiguration(); + config.getCompilationCustomizers().add(consoleImports); + config.setTargetDirectory(classCacheDirectory); + GroovyClassLoader classLoader = new GroovyClassLoader(STATIC_LOADER, config); // Update the session's ExecutionContext with fresh QueryCompiler executionContext = createExecutionContext( executionContext.getUpdateGraph(), executionContext.getOperationInitializer(), - freshClassLoader); + classLoader); // Permanently replace groovyShell with fresh shell - groovyShell = new DeephavenGroovyShell(freshClassLoader, groovyShell.getContext(), freshConfig); + groovyShell = new DeephavenGroovyShell(classLoader, groovyShell.getContext(), config); } @Override @@ -393,8 +393,11 @@ protected void evaluate(String command, String scriptName) { final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); final boolean hasRemoteSources = remoteLoader.hasConfiguredRemoteSources(); - // If we are switching from no remote sources to active remote sources, we don't know which classes in the - // cache will be sourced remotely, so we clear the entire cache before execution. + // Any time there are remote sources, we need to clear the class cache to ensure they are freshly fetched. + // 1. For cases where previous evaluate had remote sources, the cache will have already been cleared of any + // fetched sources, so nothing to do. + // 2. For cases where previous evaluate did not have remote sources, we have to clear the entire cache since + // we don't know yet what sources need to be fetched. if (!previousEvalHadRemoteSources && hasRemoteSources) { log.debug("Remote file sourcing enabled. Clearing class cache."); refreshClassLoader(null); // null = clear all cache files @@ -420,8 +423,7 @@ protected void evaluate(String command, String scriptName) { // Get list of remote resources that were used in this execution final List remoteResources = remoteLoader.getFetchedRemoteResources(); - // Scenario 2: Remote sources were fetched during execution - // Selectively clear those specific .class files and refresh classloader for next execution + // If remote sources were fetched, clear them from the cache so that they will be re-fetched on next evaluate if (!remoteResources.isEmpty()) { log.debug("Remote resources were fetched - selectively clearing cache after execution"); From 22762e3de053bc54a2ca9b82da3e7fce03d3b72c Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 24 Feb 2026 10:52:16 -0600 Subject: [PATCH 63/78] Clean up (#DH-20578) --- .../engine/util/GroovyDeephavenSession.java | 102 +++++------------- 1 file changed, 25 insertions(+), 77 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 7240f700425..2e111cf41fc 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -350,17 +350,12 @@ private void refreshClassLoader(@Nullable Set classPathsToClear) { throw new IllegalStateException("Cannot refresh classloader: cache directory does not exist"); } - if (classPathsToClear == null) { - // Clear all cached .class files - deleteClassFiles(classCacheDirectory); - } - else if(classPathsToClear.isEmpty()) { + if (classPathsToClear != null && classPathsToClear.isEmpty()) { throw new IllegalArgumentException("classPathsToClear cannot be empty"); } - else { - // Selectively clear only specified .class files - deleteMatchingClassFiles(classCacheDirectory, classPathsToClear); - } + + // Clear cached .class files (all if classPathsToClear is null, otherwise only specified files) + deleteClassFiles(classCacheDirectory, classPathsToClear); // Create a fresh GroovyClassLoader CompilerConfiguration config = new CompilerConfiguration(); @@ -805,35 +800,19 @@ private static boolean isAnInteger(final String s) { } /** - * Delete .class files from the cache directory to force recompilation. + * Recursively delete .class files from the cache directory to force recompilation. * This is necessary because GroovyClassLoader can reload .class files from disk even after clearCache(). + * + * @param directory the directory to recursively search for .class files (initially classCacheDirectory, then subdirectories during recursion) + * @param classFilePaths optional set of specific class file paths to delete. If null, all .class files are deleted. + * Must not be empty if provided. + * @throws IllegalArgumentException if classFilePaths is non-null but empty */ - private static void deleteClassFiles(File directory) { - if (directory == null || !directory.exists() || !directory.isDirectory()) { - return; - } - - File[] files = directory.listFiles(); - if (files == null) { - return; + private void deleteClassFiles(File directory, @Nullable Set classFilePaths) { + if (classFilePaths != null && classFilePaths.isEmpty()) { + throw new IllegalArgumentException("classFilePaths cannot be empty"); } - for (File file : files) { - if (file.isDirectory()) { - deleteClassFiles(file); - } else if (file.getName().endsWith(".class")) { - if (!file.delete()) { - log.warn("Failed to delete cached class file: " + file.getAbsolutePath()); - } - } - } - } - - - /** - * Recursively delete .class files that match the given set of paths. - */ - private static void deleteMatchingClassFiles(File directory, Set classFilePaths) { if (directory == null || !directory.exists() || !directory.isDirectory()) { return; } @@ -845,13 +824,20 @@ private static void deleteMatchingClassFiles(File directory, Set classFi for (File file : files) { if (file.isDirectory()) { - deleteMatchingClassFiles(file, classFilePaths); + deleteClassFiles(file, classFilePaths); } else if (file.getName().endsWith(".class")) { - // Get relative path from classCacheDirectory - String relativePath = getRelativePath(file); - if (classFilePaths.contains(relativePath)) { + boolean shouldDelete = (classFilePaths == null); + if (!shouldDelete) { + String relativePath = classCacheDirectory.toPath().relativize(file.toPath()) + .toString().replace(File.separatorChar, '/'); + shouldDelete = classFilePaths.contains(relativePath); + } + + if (shouldDelete) { if (file.delete()) { - log.info("Deleted cached .class file for remote source: " + relativePath); + if (classFilePaths != null) { + log.info("Deleted cached .class file for remote source: " + file.getPath()); + } } else { log.warn("Failed to delete cached class file: " + file.getAbsolutePath()); } @@ -860,44 +846,6 @@ private static void deleteMatchingClassFiles(File directory, Set classFi } } - /** - * Get a relative path string from a file by walking up to find a reasonable base. - * This is a simple implementation that uses forward slashes. - */ - private static String getRelativePath(File file) { - // Simple approach: get the path and look for package-like structure - String path = file.getPath(); - // Replace backslashes with forward slashes for consistency - path = path.replace('\\', '/'); - - // Try to find a reasonable starting point by looking for common package prefixes - // For now, just return the path segments that look like packages - int lastSeparator = path.lastIndexOf('/'); - if (lastSeparator > 0) { - // Look backwards to find what looks like a package structure - // This is a simple heuristic - could be improved - String[] segments = path.split("/"); - StringBuilder result = new StringBuilder(); - boolean inPackage = false; - for (int i = 0; i < segments.length; i++) { - String segment = segments[i]; - // Start capturing when we see a lowercase segment (likely package name) - if (!inPackage && segment.length() > 0 && Character.isLowerCase(segment.charAt(0))) { - inPackage = true; - } - if (inPackage) { - if (result.length() > 0) { - result.append('/'); - } - result.append(segment); - } - } - if (result.length() > 0) { - return result.toString(); - } - } - return file.getName(); - } @Override protected GroovySnapshot emptySnapshot() { From 1a8a97c5c804691dcaef20c1a0ecd66c87703441 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 24 Feb 2026 11:14:25 -0600 Subject: [PATCH 64/78] cleanup (#DH-20578) --- .../deephaven/engine/util/GroovyDeephavenSession.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 2e111cf41fc..e7ca896c622 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -354,8 +354,7 @@ private void refreshClassLoader(@Nullable Set classPathsToClear) { throw new IllegalArgumentException("classPathsToClear cannot be empty"); } - // Clear cached .class files (all if classPathsToClear is null, otherwise only specified files) - deleteClassFiles(classCacheDirectory, classPathsToClear); + deleteCachedClassFiles(classCacheDirectory, classPathsToClear); // Create a fresh GroovyClassLoader CompilerConfiguration config = new CompilerConfiguration(); @@ -808,12 +807,12 @@ private static boolean isAnInteger(final String s) { * Must not be empty if provided. * @throws IllegalArgumentException if classFilePaths is non-null but empty */ - private void deleteClassFiles(File directory, @Nullable Set classFilePaths) { + private void deleteCachedClassFiles(@NotNull File directory, @Nullable Set classFilePaths) { if (classFilePaths != null && classFilePaths.isEmpty()) { throw new IllegalArgumentException("classFilePaths cannot be empty"); } - if (directory == null || !directory.exists() || !directory.isDirectory()) { + if (!directory.exists() || !directory.isDirectory()) { return; } @@ -824,7 +823,7 @@ private void deleteClassFiles(File directory, @Nullable Set classFilePat for (File file : files) { if (file.isDirectory()) { - deleteClassFiles(file, classFilePaths); + deleteCachedClassFiles(file, classFilePaths); } else if (file.getName().endsWith(".class")) { boolean shouldDelete = (classFilePaths == null); if (!shouldDelete) { @@ -846,7 +845,6 @@ private void deleteClassFiles(File directory, @Nullable Set classFilePat } } - @Override protected GroovySnapshot emptySnapshot() { return new GroovySnapshot(Collections.emptyMap()); From 3c67bff49a1e9dfd16ee7032469cf31e68ee8edf Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 24 Feb 2026 13:01:59 -0600 Subject: [PATCH 65/78] cleanup (#DH-20578) --- .../engine/util/GroovyDeephavenSession.java | 104 +++++++++++------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index e7ca896c622..c4291df8414 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -51,6 +51,7 @@ import org.codehaus.groovy.tools.GroovyClass; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.jspecify.annotations.NonNull; import java.io.File; import java.io.IOException; @@ -338,23 +339,23 @@ protected T getVariable(String name) { * Clears cached .class files and refreshes the GroovyClassLoader and associated shell. * This ensures that classes will be recompiled from source on the next execution. * - * @param classPathsToClear set of specific class file paths to delete (e.g., "test2/notebook/MyNotebook.class"). - * If null, all .class files in the cache directory will be deleted. - * Must not be empty. + * @param groovyBasePaths list of base class paths from Groovy sources (e.g., "test2/notebook/MyNotebook"). + * If null, all .class files in the cache directory will be deleted. + * Must not be empty if provided. * @throws IllegalStateException if the cache directory does not exist - * @throws IllegalArgumentException if classPathsToClear is empty + * @throws IllegalArgumentException if basePaths is empty */ - private void refreshClassLoader(@Nullable Set classPathsToClear) { + private void refreshClassLoader(@Nullable List groovyBasePaths) { // If no cache directory exists, this is an error - we shouldn't be trying to refresh without a cache if (classCacheDirectory == null || !classCacheDirectory.exists()) { throw new IllegalStateException("Cannot refresh classloader: cache directory does not exist"); } - if (classPathsToClear != null && classPathsToClear.isEmpty()) { - throw new IllegalArgumentException("classPathsToClear cannot be empty"); + if (groovyBasePaths != null && groovyBasePaths.isEmpty()) { + throw new IllegalArgumentException("groovyBasePaths cannot be empty"); } - deleteCachedClassFiles(classCacheDirectory, classPathsToClear); + deleteCachedClassFiles(classCacheDirectory, groovyBasePaths); // Create a fresh GroovyClassLoader CompilerConfiguration config = new CompilerConfiguration(); @@ -415,29 +416,33 @@ protected void evaluate(String command, String scriptName) { } } finally { // Get list of remote resources that were used in this execution - final List remoteResources = remoteLoader.getFetchedRemoteResources(); - - // If remote sources were fetched, clear them from the cache so that they will be re-fetched on next evaluate - if (!remoteResources.isEmpty()) { - log.debug("Remote resources were fetched - selectively clearing cache after execution"); - - // Convert remote resource names to class file paths - // e.g., "test2/notebook/MyNotebook.groovy" -> "test2/notebook/MyNotebook.class" - Set classFilePaths = new HashSet<>(); - for (String resource : remoteResources) { - if (resource.endsWith(".groovy")) { - String classPath = resource.substring(0, resource.length() - 7) + ".class"; - classFilePaths.add(classPath); - log.debug("Will delete .class file for remote source: " + classPath); - } - } + List groovyBasePaths = getGroovyBasePaths(remoteLoader.getFetchedRemoteResources()); + + // If remote .groovy sources were fetched, clear associated classes from the cache so that they will be + // re-fetched on next evaluate + if (!groovyBasePaths.isEmpty()) { + log.debug("Clearing remote .groovy sources from class cache"); + refreshClassLoader(groovyBasePaths); + } - refreshClassLoader(classFilePaths); + remoteLoader.clearFetchedResourcesTracking(); + } + } - // Always clear tracking after selective cleanup to prevent accumulation - remoteLoader.clearFetchedResourcesTracking(); + /** + * Extracts base paths from remote resources by filtering to .groovy files and stripping their extensions. + * + * @param remoteResources list of remote resource paths that were fetched + * @return list of base paths (without .groovy extension) for all groovy files in the input + */ + private static @NonNull List getGroovyBasePaths(List remoteResources) { + List groovyBasePaths = new ArrayList<>(); + for (String resource : remoteResources) { + if (resource.endsWith(".groovy")) { + groovyBasePaths.add(resource.substring(0, resource.length() - 7)); } } + return groovyBasePaths; } private RuntimeException wrapAndRewriteStackTrace(String scriptName, String currentScriptName, Exception e, @@ -798,18 +803,43 @@ private static boolean isAnInteger(final String s) { } } + /** + * Predicate function to determine if a cached class file path matches any of the base class paths. + * Handles the fact that a single .groovy file can generate multiple .class files + * (main class, inner classes, closures, anonymous classes, etc.). + * + * @param cachedClassPath the relative path of a cached class file (e.g., "test2/notebook/MyNotebook$InnerClass.class") + * @param basePaths list of base class paths (e.g., "test2/notebook/MyNotebook") + * @return true if the cached class file should be deleted because it matches one of the base paths + */ + private boolean isMatchingClassFile(String cachedClassPath, List basePaths) { + // Check if this class file matches any base class path + // Handle: MyNotebook.class, MyNotebook$InnerClass.class, MyNotebook$_closure1.class, etc. + for (String basePath : basePaths) { + String basePathWithClass = basePath + ".class"; + String basePathWithDollar = basePath + "$"; + + if (cachedClassPath.equals(basePathWithClass) || + (cachedClassPath.startsWith(basePathWithDollar) && cachedClassPath.endsWith(".class"))) { + return true; + } + } + return false; + } + /** * Recursively delete .class files from the cache directory to force recompilation. * This is necessary because GroovyClassLoader can reload .class files from disk even after clearCache(). * * @param directory the directory to recursively search for .class files (initially classCacheDirectory, then subdirectories during recursion) - * @param classFilePaths optional set of specific class file paths to delete. If null, all .class files are deleted. - * Must not be empty if provided. - * @throws IllegalArgumentException if classFilePaths is non-null but empty + * @param groovyBasePaths optional list of base .groovy paths to find classes for. If null, all .class files are deleted. + * If non-null, only class files matching these base paths are deleted. + * Must not be empty if provided. + * @throws IllegalArgumentException if basePaths is non-null but empty */ - private void deleteCachedClassFiles(@NotNull File directory, @Nullable Set classFilePaths) { - if (classFilePaths != null && classFilePaths.isEmpty()) { - throw new IllegalArgumentException("classFilePaths cannot be empty"); + private void deleteCachedClassFiles(@NotNull File directory, @Nullable List groovyBasePaths) { + if (groovyBasePaths != null && groovyBasePaths.isEmpty()) { + throw new IllegalArgumentException("basePaths cannot be empty"); } if (!directory.exists() || !directory.isDirectory()) { @@ -823,18 +853,18 @@ private void deleteCachedClassFiles(@NotNull File directory, @Nullable Set Date: Tue, 24 Feb 2026 13:19:07 -0600 Subject: [PATCH 66/78] cleanup (#DH-20578) --- .../engine/util/GroovyDeephavenSession.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index c4291df8414..8744e37f1af 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -804,23 +804,26 @@ private static boolean isAnInteger(final String s) { } /** - * Predicate function to determine if a cached class file path matches any of the base class paths. + * Determine if a cached class file path matches any of the base class paths. * Handles the fact that a single .groovy file can generate multiple .class files * (main class, inner classes, closures, anonymous classes, etc.). * * @param cachedClassPath the relative path of a cached class file (e.g., "test2/notebook/MyNotebook$InnerClass.class") - * @param basePaths list of base class paths (e.g., "test2/notebook/MyNotebook") + * @param groovyBasePaths list of base class paths (e.g., "test2/notebook/MyNotebook") * @return true if the cached class file should be deleted because it matches one of the base paths */ - private boolean isMatchingClassFile(String cachedClassPath, List basePaths) { + private boolean isMatchingClassFile(@NotNull String cachedClassPath, @NotNull List groovyBasePaths) { + if (!cachedClassPath.endsWith(".class")) { + return false; + } + // Check if this class file matches any base class path // Handle: MyNotebook.class, MyNotebook$InnerClass.class, MyNotebook$_closure1.class, etc. - for (String basePath : basePaths) { + for (String basePath : groovyBasePaths) { String basePathWithClass = basePath + ".class"; String basePathWithDollar = basePath + "$"; - if (cachedClassPath.equals(basePathWithClass) || - (cachedClassPath.startsWith(basePathWithDollar) && cachedClassPath.endsWith(".class"))) { + if (cachedClassPath.equals(basePathWithClass) || cachedClassPath.startsWith(basePathWithDollar)) { return true; } } From 8375252882da400782c754b92015e07ea7c8b4ec Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 25 Feb 2026 14:57:43 -0600 Subject: [PATCH 67/78] Simplified to an evict all strategy (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 38 ------ .../engine/util/GroovyDeephavenSession.java | 112 ++---------------- 2 files changed, 12 insertions(+), 138 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 5959eca23c4..720fbc26784 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -10,11 +10,6 @@ import java.net.URL; import java.net.URLConnection; import java.net.URLStreamHandler; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; @@ -35,9 +30,6 @@ public class RemoteFileSourceClassLoader extends ClassLoader { private static volatile RemoteFileSourceClassLoader instance; private final CopyOnWriteArrayList providers = new CopyOnWriteArrayList<>(); - // Track which remote resources have been fetched (for cache invalidation optimization) - // Using a thread-safe Set to ensure uniqueness and O(1) contains() performance - private final Set fetchedRemoteResources = Collections.synchronizedSet(new HashSet<>()); /** * Constructs a new RemoteFileSourceClassLoader with the specified parent class loader. @@ -100,34 +92,6 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { providers.remove(provider); } - /** - * Records that a remote resource has been fetched. - * This is used to track whether cache invalidation is needed. - * - * @param resourceName the name of the resource that was fetched - */ - void recordRemoteResourceFetch(String resourceName) { - fetchedRemoteResources.add(resourceName); - } - - - /** - * Returns a copy of the list of remote resources that have been fetched. - * This can be used for selective cache invalidation. - * - * @return list of fetched remote resource names - */ - public List getFetchedRemoteResources() { - return new ArrayList<>(fetchedRemoteResources); - } - - /** - * Clears the tracking of fetched remote resources. - * This should be called after cache invalidation to reset the state. - */ - public void clearFetchedResourcesTracking() { - fetchedRemoteResources.clear(); - } /** * Returns whether there are any active providers with resource paths configured. @@ -257,8 +221,6 @@ public void connect() throws IOException { .orTimeout(RESOURCE_TIMEOUT_SECONDS, TimeUnit.SECONDS) .get(); connected = true; - // Track that this remote resource was fetched - RemoteFileSourceClassLoader.getInstance().recordRemoteResourceFetch(resourceName); } catch (Exception e) { throw new IOException("Failed to fetch remote resource: " + resourceName, e); } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 8744e37f1af..0e6f0223632 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -51,7 +51,6 @@ import org.codehaus.groovy.tools.GroovyClass; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jspecify.annotations.NonNull; import java.io.File; import java.io.IOException; @@ -339,23 +338,15 @@ protected T getVariable(String name) { * Clears cached .class files and refreshes the GroovyClassLoader and associated shell. * This ensures that classes will be recompiled from source on the next execution. * - * @param groovyBasePaths list of base class paths from Groovy sources (e.g., "test2/notebook/MyNotebook"). - * If null, all .class files in the cache directory will be deleted. - * Must not be empty if provided. * @throws IllegalStateException if the cache directory does not exist - * @throws IllegalArgumentException if basePaths is empty */ - private void refreshClassLoader(@Nullable List groovyBasePaths) { + private void refreshClassLoader() { // If no cache directory exists, this is an error - we shouldn't be trying to refresh without a cache if (classCacheDirectory == null || !classCacheDirectory.exists()) { throw new IllegalStateException("Cannot refresh classloader: cache directory does not exist"); } - if (groovyBasePaths != null && groovyBasePaths.isEmpty()) { - throw new IllegalArgumentException("groovyBasePaths cannot be empty"); - } - - deleteCachedClassFiles(classCacheDirectory, groovyBasePaths); + deleteCachedClassFiles(classCacheDirectory); // Create a fresh GroovyClassLoader CompilerConfiguration config = new CompilerConfiguration(); @@ -388,14 +379,10 @@ protected void evaluate(String command, String scriptName) { final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); final boolean hasRemoteSources = remoteLoader.hasConfiguredRemoteSources(); - // Any time there are remote sources, we need to clear the class cache to ensure they are freshly fetched. - // 1. For cases where previous evaluate had remote sources, the cache will have already been cleared of any - // fetched sources, so nothing to do. - // 2. For cases where previous evaluate did not have remote sources, we have to clear the entire cache since - // we don't know yet what sources need to be fetched. - if (!previousEvalHadRemoteSources && hasRemoteSources) { - log.debug("Remote file sourcing enabled. Clearing class cache."); - refreshClassLoader(null); // null = clear all cache files + // Clear the class cache whenever remote sources are involved to ensure fresh compilation + if (previousEvalHadRemoteSources || hasRemoteSources) { + log.debug("Remote file sourcing enabled or was previously enabled. Clearing class cache."); + refreshClassLoader(); } // Update state tracker for next execution @@ -414,36 +401,9 @@ protected void evaluate(String command, String scriptName) { } catch (Exception e) { throw wrapAndRewriteStackTrace(scriptName, currentScriptName, e, lastCommand, commandPrefix); } - } finally { - // Get list of remote resources that were used in this execution - List groovyBasePaths = getGroovyBasePaths(remoteLoader.getFetchedRemoteResources()); - - // If remote .groovy sources were fetched, clear associated classes from the cache so that they will be - // re-fetched on next evaluate - if (!groovyBasePaths.isEmpty()) { - log.debug("Clearing remote .groovy sources from class cache"); - refreshClassLoader(groovyBasePaths); - } - - remoteLoader.clearFetchedResourcesTracking(); } } - /** - * Extracts base paths from remote resources by filtering to .groovy files and stripping their extensions. - * - * @param remoteResources list of remote resource paths that were fetched - * @return list of base paths (without .groovy extension) for all groovy files in the input - */ - private static @NonNull List getGroovyBasePaths(List remoteResources) { - List groovyBasePaths = new ArrayList<>(); - for (String resource : remoteResources) { - if (resource.endsWith(".groovy")) { - groovyBasePaths.add(resource.substring(0, resource.length() - 7)); - } - } - return groovyBasePaths; - } private RuntimeException wrapAndRewriteStackTrace(String scriptName, String currentScriptName, Exception e, String lastCommand, String commandPrefix) { @@ -804,47 +764,12 @@ private static boolean isAnInteger(final String s) { } /** - * Determine if a cached class file path matches any of the base class paths. - * Handles the fact that a single .groovy file can generate multiple .class files - * (main class, inner classes, closures, anonymous classes, etc.). - * - * @param cachedClassPath the relative path of a cached class file (e.g., "test2/notebook/MyNotebook$InnerClass.class") - * @param groovyBasePaths list of base class paths (e.g., "test2/notebook/MyNotebook") - * @return true if the cached class file should be deleted because it matches one of the base paths - */ - private boolean isMatchingClassFile(@NotNull String cachedClassPath, @NotNull List groovyBasePaths) { - if (!cachedClassPath.endsWith(".class")) { - return false; - } - - // Check if this class file matches any base class path - // Handle: MyNotebook.class, MyNotebook$InnerClass.class, MyNotebook$_closure1.class, etc. - for (String basePath : groovyBasePaths) { - String basePathWithClass = basePath + ".class"; - String basePathWithDollar = basePath + "$"; - - if (cachedClassPath.equals(basePathWithClass) || cachedClassPath.startsWith(basePathWithDollar)) { - return true; - } - } - return false; - } - - /** - * Recursively delete .class files from the cache directory to force recompilation. + * Recursively delete all .class files from the cache directory to force recompilation. * This is necessary because GroovyClassLoader can reload .class files from disk even after clearCache(). * - * @param directory the directory to recursively search for .class files (initially classCacheDirectory, then subdirectories during recursion) - * @param groovyBasePaths optional list of base .groovy paths to find classes for. If null, all .class files are deleted. - * If non-null, only class files matching these base paths are deleted. - * Must not be empty if provided. - * @throws IllegalArgumentException if basePaths is non-null but empty + * @param directory the directory to recursively search for .class files */ - private void deleteCachedClassFiles(@NotNull File directory, @Nullable List groovyBasePaths) { - if (groovyBasePaths != null && groovyBasePaths.isEmpty()) { - throw new IllegalArgumentException("basePaths cannot be empty"); - } - + private void deleteCachedClassFiles(@NotNull File directory) { if (!directory.exists() || !directory.isDirectory()) { return; } @@ -856,23 +781,10 @@ private void deleteCachedClassFiles(@NotNull File directory, @Nullable List Date: Wed, 25 Feb 2026 15:40:39 -0600 Subject: [PATCH 68/78] comment (#DH-20578) --- .../java/io/deephaven/engine/util/GroovyDeephavenSession.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 0e6f0223632..72ed598feec 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -379,7 +379,9 @@ protected void evaluate(String command, String scriptName) { final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); final boolean hasRemoteSources = remoteLoader.hasConfiguredRemoteSources(); - // Clear the class cache whenever remote sources are involved to ensure fresh compilation + // If previous run had remote sources enabled, we need to clear the cache in case anything has changed. + // If current run has remote sources enabled, we also need to clear the cache so that cached local classes have + // opportunity to be remotely fetched. If remote sources aren't involved, no need to clear the cache. if (previousEvalHadRemoteSources || hasRemoteSources) { log.debug("Remote file sourcing enabled or was previously enabled. Clearing class cache."); refreshClassLoader(); From a5f3af50f3883d43faad41b8bfe4b6f30ca7cfa4 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 25 Feb 2026 15:48:13 -0600 Subject: [PATCH 69/78] removed unused method (#DH-20578) --- .../engine/util/RemoteFileSourceClassLoader.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 720fbc26784..3b62f482afe 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -92,22 +92,6 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { providers.remove(provider); } - - /** - * Returns whether there are any active providers with resource paths configured. - * This indicates that remote sources are available and may be used. - * - * @return true if any provider is active and has resources it can source, false otherwise - */ - public boolean hasActiveProviders() { - for (RemoteFileSourceProvider candidate : providers) { - if (candidate.isActive()) { - return true; - } - } - return false; - } - /** * Returns whether there are any active providers with non-empty resource paths configured. * This indicates that remote sources are actually configured, not just that the execution context is set. From 24b0d29998d70e7c0f55e2e4d8b3e3f93a172f71 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 25 Feb 2026 16:17:03 -0600 Subject: [PATCH 70/78] comments (#DH-20578) --- .../RemoteFileSourceCommandResolver.java | 2 +- plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index 149edf63b13..ef5e489c6ef 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -101,7 +101,7 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl } // Export a plugin-specific PluginMarker. Plugins using PluginMarker should check - // marker.getPluginName() in isType() to prevent conflicts when multiple plugins share PluginMarker. + // marker.getPluginName() in isType() to prevent conflicts with markers for other plugins. session.newExport(resultTicket, "RemoteFileSourcePluginFetchRequest.resultTicket") .submit(() -> PluginMarker.forPluginName(pluginName)); diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index 2eaa3dcb2f8..cebe211aca1 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -7,7 +7,7 @@ import java.util.concurrent.ConcurrentHashMap; /** - * A generic marker object for plugin exports that can be shared across multiple plugin types. + * A generic marker object for plugin exports that can be used by multiple plugin types. *

    * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() * returns the FIRST plugin where isType() returns true. Without plugin-specific identification From 168e776d74912d854f58f3695a7b9f410ec85112 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 25 Feb 2026 17:20:09 -0600 Subject: [PATCH 71/78] spotless (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 42 ++++++---- .../engine/util/RemoteFileSourceProvider.java | 11 ++- .../engine/util/AbstractScriptSession.java | 4 +- .../engine/util/GroovyDeephavenSession.java | 14 ++-- .../RemoteFileSourceCommandResolver.java | 45 +++++----- .../RemoteFileSourceMessageStream.java | 84 +++++++++---------- .../RemoteFileSourcePlugin.java | 19 ++--- ...ileSourceTicketResolverFactoryService.java | 8 +- .../deephaven/plugin/type/PluginMarker.java | 16 ++-- .../web/client/api/JsProtobufUtils.java | 84 +++++++++---------- .../JsRemoteFileSourceService.java | 34 ++++---- .../ResourceContentUnion.java | 2 +- .../RemoteFileSourceClientRequest.java | 4 +- .../RemoteFileSourceMetaRequest.java | 2 +- .../RemoteFileSourceMetaResponse.java | 2 +- .../RemoteFileSourcePluginFetchRequest.java | 2 +- .../RemoteFileSourceServerRequest.java | 2 +- .../SetExecutionContextRequest.java | 2 +- .../SetExecutionContextResponse.java | 2 +- .../RequestCase.java | 4 +- .../RequestCase.java | 2 +- 21 files changed, 193 insertions(+), 192 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 3b62f482afe..75121a76ae5 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.engine.util; @@ -17,11 +17,12 @@ * A custom ClassLoader that fetches source files from remote clients via registered RemoteFileSourceProvider instances. * This is designed to support Groovy script imports where the source files are provided by remote clients. * - *

    When a resource is requested (e.g., for a Groovy import), this class loader: + *

    + * When a resource is requested (e.g., for a Groovy import), this class loader: *

      - *
    1. Checks registered providers to see if they can source the resource
      2. - *
    2. Returns a custom URL with protocol "remotefile://" if a provider can handle it
      3. - *
    3. When that URL is opened, fetches the resource bytes from the provider
      4. + *
    4. Checks registered providers to see if they can source the resource
      5. + *
    5. Returns a custom URL with protocol "remotefile://" if a provider can handle it
      6. + *
    6. When that URL is opened, fetches the resource bytes from the provider
      7. *
    */ public class RemoteFileSourceClassLoader extends ClassLoader { @@ -43,8 +44,9 @@ private RemoteFileSourceClassLoader(ClassLoader parent) { /** * Initializes the singleton RemoteFileSourceClassLoader instance with the specified parent class loader. * - *

    This method must be called exactly once before any calls to {@link #getInstance()}. The method is - * synchronized to prevent race conditions when multiple threads attempt initialization. + *

    + * This method must be called exactly once before any calls to {@link #getInstance()}. The method is synchronized to + * prevent race conditions when multiple threads attempt initialization. * * @param parent the parent class loader for delegation * @return the newly created singleton instance @@ -62,7 +64,8 @@ public static synchronized RemoteFileSourceClassLoader initialize(ClassLoader pa /** * Returns the singleton instance of the RemoteFileSourceClassLoader. * - *

    This method requires that {@link #initialize(ClassLoader)} has been called first. + *

    + * This method requires that {@link #initialize(ClassLoader)} has been called first. * * @return the singleton instance * @throws IllegalStateException if the instance has not yet been initialized via {@link #initialize(ClassLoader)} @@ -93,8 +96,8 @@ public void unregisterProvider(RemoteFileSourceProvider provider) { } /** - * Returns whether there are any active providers with non-empty resource paths configured. - * This indicates that remote sources are actually configured, not just that the execution context is set. + * Returns whether there are any active providers with non-empty resource paths configured. This indicates that + * remote sources are actually configured, not just that the execution context is set. * * @return true if any provider is active and has resource paths configured, false otherwise */ @@ -110,9 +113,10 @@ public boolean hasConfiguredRemoteSources() { /** * Gets the resource with the specified name by checking registered providers. * - *

    This method iterates through all registered providers to see if any can source the requested resource. - * If a provider can handle the resource, a custom URL with protocol "remotefile://" is returned. - * If no provider can handle the resource, the request is delegated to the parent class loader. + *

    + * This method iterates through all registered providers to see if any can source the requested resource. If a + * provider can handle the resource, a custom URL with protocol "remotefile://" is returned. If no provider can + * handle the resource, the request is delegated to the parent class loader. * * @param name the resource name * @return a URL for reading the resource, or null if the resource could not be found @@ -192,8 +196,9 @@ private static class RemoteFileURLConnection extends URLConnection { /** * Opens a connection to the resource by requesting it from the provider. * - *

    This method fetches the resource bytes from the provider with a timeout of - * {@value #RESOURCE_TIMEOUT_SECONDS} seconds. If already connected, this method does nothing. + *

    + * This method fetches the resource bytes from the provider with a timeout of {@value #RESOURCE_TIMEOUT_SECONDS} + * seconds. If already connected, this method does nothing. * * @throws IOException if the connection fails or times out */ @@ -214,9 +219,10 @@ public void connect() throws IOException { /** * Returns an input stream that reads from this connection's resource. * - *

    This method calls {@link #connect()} to ensure the connection is established and resource bytes are - * fetched from the provider. The method then verifies that content has been successfully downloaded before - * creating the input stream. + *

    + * This method calls {@link #connect()} to ensure the connection is established and resource bytes are fetched + * from the provider. The method then verifies that content has been successfully downloaded before creating the + * input stream. * * @return an input stream that reads from the fetched resource bytes * @throws IOException if the connection or content download fails or if the resource has no content diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index 08fe4f9240b..ebcb6a3310d 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -1,14 +1,13 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.engine.util; import java.util.concurrent.CompletableFuture; /** - * Interface for providing remote resources to the ClassLoader. - * Plugins can implement this interface and register with RemoteFileSourceClassLoader - * to provide resources from remote sources. + * Interface for providing remote resources to the ClassLoader. Plugins can implement this interface and register with + * RemoteFileSourceClassLoader to provide resources from remote sources. */ public interface RemoteFileSourceProvider { /** @@ -27,8 +26,8 @@ public interface RemoteFileSourceProvider { boolean isActive(); /** - * Check if this provider has any resource paths configured. - * A provider can be active (execution context set) but have no resource paths configured. + * Check if this provider has any resource paths configured. A provider can be active (execution context set) but + * have no resource paths configured. * * @return true if this provider has resource paths configured, false otherwise */ diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index d3d3e220b39..8be2deec8df 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -101,8 +101,8 @@ protected AbstractScriptSession( } /** - * DH-20578: Creates an ExecutionContext with a QueryCompiler based on the provided ClassLoader. - * This allows updating the ExecutionContext when fresh ClassLoaders are needed (e.g., for remote file sources). + * DH-20578: Creates an ExecutionContext with a QueryCompiler based on the provided ClassLoader. This allows + * updating the ExecutionContext when fresh ClassLoaders are needed (e.g., for remote file sources). * * @param updateGraph the update graph for the context * @param operationInitializer the operation initializer for the context diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 72ed598feec..2a1beb45612 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -96,7 +96,8 @@ public class GroovyDeephavenSession extends AbstractScriptSession mapping = new ConcurrentHashMap<>(); @Override @@ -335,8 +336,8 @@ protected T getVariable(String name) { } /** - * Clears cached .class files and refreshes the GroovyClassLoader and associated shell. - * This ensures that classes will be recompiled from source on the next execution. + * Clears cached .class files and refreshes the GroovyClassLoader and associated shell. This ensures that classes + * will be recompiled from source on the next execution. * * @throws IllegalStateException if the cache directory does not exist */ @@ -766,8 +767,8 @@ private static boolean isAnInteger(final String s) { } /** - * Recursively delete all .class files from the cache directory to force recompilation. - * This is necessary because GroovyClassLoader can reload .class files from disk even after clearCache(). + * Recursively delete all .class files from the cache directory to force recompilation. This is necessary because + * GroovyClassLoader can reload .class files from disk even after clearCache(). * * @param directory the directory to recursively search for .class files */ @@ -781,6 +782,9 @@ private void deleteCachedClassFiles(@NotNull File directory) { return; } + log.debug().append("Deleting ").append(files.length).append(" files from class cache directory: ") + .append(directory.getAbsolutePath()); + for (File file : files) { if (file.isDirectory()) { deleteCachedClassFiles(file); diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java index ef5e489c6ef..5c8bef481ff 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceCommandResolver.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.remotefilesource; @@ -54,9 +54,8 @@ private static RemoteFileSourcePluginFetchRequest parseFetchRequest(final Any co } /** - * Attempts to parse ByteString data as a protobuf Any message. - * Returns null if parsing fails rather than throwing an exception, allowing callers to handle - * invalid data gracefully. + * Attempts to parse ByteString data as a protobuf Any message. Returns null if parsing fails rather than throwing + * an exception, allowing callers to handle invalid data gracefully. * * @param data the ByteString data to parse * @return the parsed Any message, or null if parsing fails @@ -70,13 +69,12 @@ private static Any parseAsAnyOrNull(final ByteString data) { } /** - * Exports the PluginMarker singleton based on the fetch request. - * The marker object is exported to the session using the result ticket specified in the request, - * and flight info is returned containing the endpoint for accessing it. + * Exports the PluginMarker singleton based on the fetch request. The marker object is exported to the session using + * the result ticket specified in the request, and flight info is returned containing the endpoint for accessing it. * - *

    Note: This exports a PluginMarker for the specified plugin name. Plugin-specific routing - * is handled by TypedTicket.type in the ConnectRequest phase, which is validated against - * the plugin's name() method. + *

    + * Note: This exports a PluginMarker for the specified plugin name. Plugin-specific routing is handled by + * TypedTicket.type in the ConnectRequest phase, which is validated against the plugin's name() method. * * @param session the session state for the current request * @param descriptor the flight descriptor containing the command @@ -85,8 +83,8 @@ private static Any parseAsAnyOrNull(final ByteString data) { * @throws StatusRuntimeException if the request doesn't contain a valid result ID ticket or plugin name */ private static SessionState.ExportObject fetchPlugin(@Nullable final SessionState session, - final Flight.FlightDescriptor descriptor, - final RemoteFileSourcePluginFetchRequest request) { + final Flight.FlightDescriptor descriptor, + final RemoteFileSourcePluginFetchRequest request) { final Ticket resultTicket = request.getResultId(); final boolean hasResultId = !resultTicket.getTicket().isEmpty(); if (!hasResultId) { @@ -120,16 +118,16 @@ private static SessionState.ExportObject fetchPlugin(@Nullabl } /** - * Resolves a flight descriptor to flight info for remote file source commands. - * Handles RemoteFileSourcePluginFetchRequest commands by parsing the descriptor and delegating to the - * appropriate handler method. + * Resolves a flight descriptor to flight info for remote file source commands. Handles + * RemoteFileSourcePluginFetchRequest commands by parsing the descriptor and delegating to the appropriate handler + * method. * * @param session the session state for the current request * @param descriptor the flight descriptor containing the command * @param logId the log identifier for tracking * @return a FlightInfo export object for the requested command - * @throws StatusRuntimeException if session is null (UNAUTHENTICATED), the command cannot be parsed, - * or the command type URL is not recognized + * @throws StatusRuntimeException if session is null (UNAUTHENTICATED), the command cannot be parsed, or the command + * type URL is not recognized */ @Override public SessionState.ExportObject flightInfoFor(@Nullable final SessionState session, @@ -158,7 +156,8 @@ public SessionState.ExportObject flightInfoFor(@Nullable fina /** * Visits all flight info that this resolver exposes. * - *

    Not implemented: This resolver does not expose any flight info via list flights. + *

    + * Not implemented: This resolver does not expose any flight info via list flights. */ @Override public void forAllFlightInfo(@Nullable final SessionState session, final Consumer visitor) { @@ -177,8 +176,8 @@ public String getLogNameFor(final ByteBuffer ticket, final String logId) { } /** - * Determines if this resolver is responsible for handling the given command descriptor. - * This resolver handles commands with type URL matching RemoteFileSourcePluginFetchRequest. + * Determines if this resolver is responsible for handling the given command descriptor. This resolver handles + * commands with type URL matching RemoteFileSourcePluginFetchRequest. * * @param descriptor the flight descriptor containing the command * @return true if this resolver handles the command, false otherwise @@ -247,7 +246,8 @@ public SessionState.ExportObject resolve(@Nullable final SessionState ses /** * Sets the ticket router for this resolver. * - *

    Not implemented: This resolver does not need access to the ticket router. + *

    + * Not implemented: This resolver does not need access to the ticket router. */ @Override public void setTicketRouter(TicketRouter ticketRouter) { @@ -255,8 +255,7 @@ public void setTicketRouter(TicketRouter ticketRouter) { } /** - * Returns the ticket route byte for this resolver. - * This resolver does not use ticket-based routing, so returns 0. + * Returns the ticket route byte for this resolver. This resolver does not use ticket-based routing, so returns 0. * * @return 0, indicating no ticket routing */ diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 5d9a5be2691..6fc2681cee7 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.remotefilesource; @@ -24,20 +24,18 @@ import java.util.concurrent.ConcurrentHashMap; /** - * Message stream implementation for RemoteFileSource bidirectional communication. - * Each instance represents a file source provider for one client connection and implements - * RemoteFileSourceProvider so it can be registered with the RemoteFileSourceClassLoader. - * Only one MessageStream can be "active" at a time (determined by the execution context). + * Message stream implementation for RemoteFileSource bidirectional communication. Each instance represents a file + * source provider for one client connection and implements RemoteFileSourceProvider so it can be registered with the + * RemoteFileSourceClassLoader. Only one MessageStream can be "active" at a time (determined by the execution context). * The RemoteFileSourceClassLoader checks isActive() on each registered provider to find the active one. */ public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, RemoteFileSourceProvider { private static final Logger log = LoggerFactory.getLogger(RemoteFileSourceMessageStream.class); /** - * The current execution context containing the active message stream and configuration. - * Null when no execution context is active. - * Used by this class's isActive() and canSourceResource() methods to determine if this - * provider should handle resource requests from RemoteFileSourceClassLoader. + * The current execution context containing the active message stream and configuration. Null when no execution + * context is active. Used by this class's isActive() and canSourceResource() methods to determine if this provider + * should handle resource requests from RemoteFileSourceClassLoader. */ private static volatile RemoteFileSourceExecutionContext executionContext; @@ -46,8 +44,8 @@ public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, private final Map> pendingRequests = new ConcurrentHashMap<>(); /** - * Creates a new RemoteFileSourceMessageStream for the given connection. - * Automatically registers this instance as a provider with the RemoteFileSourceClassLoader. + * Creates a new RemoteFileSourceMessageStream for the given connection. Automatically registers this instance as a + * provider with the RemoteFileSourceClassLoader. * * @param connection the message stream connection to the client */ @@ -58,9 +56,8 @@ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) } /** - * Determines if this provider can source the specified resource. - * Only returns true if this message stream is active, the resource is a .groovy file, - * and the resource path matches one of the configured resource paths. + * Determines if this provider can source the specified resource. Only returns true if this message stream is + * active, the resource is a .groovy file, and the resource path matches one of the configured resource paths. * * @param resourceName the name of the resource to check * @return true if this provider can source the resource, false otherwise @@ -95,9 +92,8 @@ public boolean canSourceResource(String resourceName) { } /** - * Requests a resource from the remote client. - * Sends a request to the client and returns a future that will be completed when the client responds. - * Only services requests if this message stream is active. + * Requests a resource from the remote client. Sends a request to the client and returns a future that will be + * completed when the client responds. Only services requests if this message stream is active. * * @param resourceName the name of the resource to request * @return a CompletableFuture that will contain the resource bytes when available, or null if inactive @@ -121,15 +117,15 @@ public CompletableFuture requestResource(String resourceName) { // Build RemoteFileSourceMetaRequest proto RemoteFileSourceMetaRequest metaRequest = RemoteFileSourceMetaRequest.newBuilder() - .setResourceName(resourceName) - .build(); + .setResourceName(resourceName) + .build(); // Wrap in RemoteFileSourceServerRequest (server→client) RemoteFileSourceServerRequest message = RemoteFileSourceServerRequest.newBuilder() - .setRequestId(requestId) - .setMetaRequest(metaRequest) - .build(); + .setRequestId(requestId) + .setMetaRequest(metaRequest) + .build(); ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); @@ -146,8 +142,8 @@ public CompletableFuture requestResource(String resourceName) { } /** - * Checks if this message stream is currently active. - * A message stream is active when the execution context is set and this instance is the active stream. + * Checks if this message stream is currently active. A message stream is active when the execution context is set + * and this instance is the active stream. * * @return true if this message stream is active, false otherwise */ @@ -174,16 +170,18 @@ public boolean hasConfiguredResources() { /** * Sets the execution context with the active message stream and resource paths. * - *

    This static method establishes which message stream instance should be considered "active" for - * resource requests, and which resource paths should be resolved from that remote source. Only one - * execution context can be active at a time across all instances. + *

    + * This static method establishes which message stream instance should be considered "active" for resource requests, + * and which resource paths should be resolved from that remote source. Only one execution context can be active at + * a time across all instances. * - *

    In multi-client scenarios (Community Core), this ensures that only the - * message stream for the currently executing script is active, preventing resource requests from - * being serviced by the wrong client connection. + *

    + * In multi-client scenarios (Community Core), this ensures that only the message stream for the currently executing + * script is active, preventing resource requests from being serviced by the wrong client connection. * - *

    Typical Usage: Called at the beginning of script execution to establish which .groovy - * files should be sourced from the remote client rather than the local classpath. + *

    + * Typical Usage: Called at the beginning of script execution to establish which .groovy files should be + * sourced from the remote client rather than the local classpath. * * @param messageStream the message stream to set as active (must not be null) * @param resourcePaths list of resource paths (e.g., "package/MyScript.groovy") to resolve from remote source @@ -210,8 +208,7 @@ public static void clearExecutionContext() { } /** - * Handles incoming data from the client. - * Parses RemoteFileSourceClientRequest messages and processes meta responses + * Handles incoming data from the client. Parses RemoteFileSourceClientRequest messages and processes meta responses * or execution context updates from the client. * * @param payload the message payload containing the protobuf data @@ -228,7 +225,8 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun if (message.hasMetaResponse()) { handleMetaResponse(message.getRequestId(), message.getMetaResponse()); } else if (message.hasSetExecutionContext()) { - handleSetExecutionContext(message.getRequestId(), message.getSetExecutionContext().getResourcePathsList()); + handleSetExecutionContext(message.getRequestId(), + message.getSetExecutionContext().getResourcePathsList()); } else { log.error().append("Received unknown message type from client").endl(); throw new ObjectCommunicationException("Received unknown message type from client"); @@ -302,9 +300,9 @@ private void sendExecutionContextAcknowledgment(String requestId) { } /** - * Handles cleanup when the message stream is closed. - * Unregisters this provider from the RemoteFileSourceClassLoader, clears the execution context if this was active, - * and cancels all pending resource requests. + * Handles cleanup when the message stream is closed. Unregisters this provider from the + * RemoteFileSourceClassLoader, clears the execution context if this was active, and cancels all pending resource + * requests. */ @Override public void onClose() { @@ -336,15 +334,15 @@ private void registerWithClassLoader() { private void unregisterFromClassLoader() { RemoteFileSourceClassLoader classLoader = RemoteFileSourceClassLoader.getInstance(); classLoader.unregisterProvider(this); - log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader").endl(); + log.info().append("Unregistered RemoteFileSourceMessageStream provider from RemoteFileSourceClassLoader") + .endl(); } /** - * Encapsulates the execution context for remote file source operations. - * This includes the currently active message stream and the resource paths - * that should be resolved from the remote source. - * This class is immutable - a new instance is created each time the context changes. + * Encapsulates the execution context for remote file source operations. This includes the currently active message + * stream and the resource paths that should be resolved from the remote source. This class is immutable - a new + * instance is created each time the context changes. */ public static class RemoteFileSourceExecutionContext { private final RemoteFileSourceMessageStream activeMessageStream; diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 92107412fba..06f8f7cff01 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.remotefilesource; @@ -12,17 +12,16 @@ import java.nio.ByteBuffer; /** - * ObjectType plugin for remote file sources. This plugin is registered via @AutoService - * and handles creation of RemoteFileSourceMessageStream connections for bidirectional - * communication with clients. + * ObjectType plugin for remote file sources. This plugin is registered via @AutoService and handles creation of + * RemoteFileSourceMessageStream connections for bidirectional communication with clients. *

    - * This plugin recognizes PluginMarker objects whose pluginName matches this plugin's name. - * When a connection is established, a RemoteFileSourceMessageStream is created to handle - * bidirectional message passing between client and server. + * This plugin recognizes PluginMarker objects whose pluginName matches this plugin's name. When a connection is + * established, a RemoteFileSourceMessageStream is created to handle bidirectional message passing between client and + * server. *

    - * Each RemoteFileSourceMessageStream instance registers itself as a provider with the - * RemoteFileSourceClassLoader when created and unregisters when closed. The RemoteFileSourceClassLoader - * uses isActive() to determine which registered provider should handle resource requests. + * Each RemoteFileSourceMessageStream instance registers itself as a provider with the RemoteFileSourceClassLoader when + * created and unregisters when closed. The RemoteFileSourceClassLoader uses isActive() to determine which registered + * provider should handle resource requests. */ @AutoService(ObjectType.class) public class RemoteFileSourcePlugin extends ObjectTypeBase { diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java index 204161b28a4..adbc1ab34ae 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceTicketResolverFactoryService.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.remotefilesource; @@ -8,9 +8,9 @@ import io.deephaven.server.session.TicketResolver; /** - * Factory service for creating RemoteFileSourceCommandResolver instances. - * This service is registered via @AutoService and provides ticket resolver functionality - * for handling remote file source plugin commands through the Deephaven server infrastructure. + * Factory service for creating RemoteFileSourceCommandResolver instances. This service is registered via @AutoService + * and provides ticket resolver functionality for handling remote file source plugin commands through the Deephaven + * server infrastructure. */ @AutoService(TicketResolversFromServiceLoader.Factory.class) public class RemoteFileSourceTicketResolverFactoryService implements TicketResolversFromServiceLoader.Factory { diff --git a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java index cebe211aca1..02ebd4f3589 100644 --- a/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java +++ b/plugin/src/main/java/io/deephaven/plugin/type/PluginMarker.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.plugin.type; @@ -9,13 +9,12 @@ /** * A generic marker object for plugin exports that can be used by multiple plugin types. *

    - * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() - * returns the FIRST plugin where isType() returns true. Without plugin-specific identification - * in isType(), multiple plugins using PluginMarker would conflict, and whichever is registered - * first would intercept all PluginMarker instances. + * IMPORTANT: The pluginName field is required because ObjectTypeLookup.findObjectType() returns the FIRST plugin where + * isType() returns true. Without plugin-specific identification in isType(), multiple plugins using PluginMarker would + * conflict, and whichever is registered first would intercept all PluginMarker instances. *

    - * This class maintains a single instance per pluginName - multiple calls to {@link #forPluginName(String)} - * with the same name will return the same instance. + * This class maintains a single instance per pluginName - multiple calls to {@link #forPluginName(String)} with the + * same name will return the same instance. */ public class PluginMarker { private static final Map INSTANCES = new ConcurrentHashMap<>(); @@ -46,8 +45,7 @@ public static PluginMarker forPluginName(String pluginName) { } /** - * Gets the plugin name this marker is intended for. - * This should match the ObjectType.name() of the target plugin. + * Gets the plugin name this marker is intended for. This should match the ObjectType.name() of the target plugin. * * @return the plugin name identifier */ diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java index 2d61a93523b..6ee6ad92c75 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/JsProtobufUtils.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.web.client.api; @@ -12,9 +12,9 @@ public class JsProtobufUtils { // Varint encoding constants - private static final int VARINT_CONTINUATION_BIT = 0x80; // 10000000 - indicates more bytes follow - private static final int VARINT_DATA_MASK = 0x7F; // 01111111 - extracts 7 bits of data - private static final int VARINT_BYTE_THRESHOLD = 128; // Values >= 128 require multiple bytes + private static final int VARINT_CONTINUATION_BIT = 0x80; // 10000000 - indicates more bytes follow + private static final int VARINT_DATA_MASK = 0x7F; // 01111111 - extracts 7 bits of data + private static final int VARINT_BYTE_THRESHOLD = 128; // Values >= 128 require multiple bytes private JsProtobufUtils() { // Utility class, no instantiation @@ -25,12 +25,12 @@ private JsProtobufUtils() { *

    * The google.protobuf.Any message has two fields: *

      - *
    • Field 1: type_url (string) - identifies the type of message contained
      • - *
    • Field 2: value (bytes) - the actual serialized message
      • + *
    • Field 1: type_url (string) - identifies the type of message contained
      • + *
    • Field 2: value (bytes) - the actual serialized message
      • *
    *

    - * This method manually encodes the Any message in protobuf binary format since the client-side - * JavaScript protobuf library doesn't provide Any.pack() like the server-side Java library does. + * This method manually encodes the Any message in protobuf binary format since the client-side JavaScript protobuf + * library doesn't provide Any.pack() like the server-side Java library does. * * @param typeUrl the type URL for the message (e.g., "type.googleapis.com/package.MessageName") * @param messageBytes the serialized protobuf message bytes @@ -40,8 +40,8 @@ public static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { // Protobuf tag constants for google.protobuf.Any message fields // Tag format: (field_number << 3) | wire_type // wire_type=2 means length-delimited (for strings/bytes) - final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 - final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 + final int TYPE_URL_TAG = 10; // (1 << 3) | 2 = field 1, wire type 2 + final int VALUE_TAG = 18; // (2 << 3) | 2 = field 2, wire type 2 // Encode the type_url string to UTF-8 bytes TextEncoder textEncoder = new TextEncoder(); @@ -70,9 +70,9 @@ public static Uint8Array wrapInAny(String typeUrl, Uint8Array messageBytes) { *

    * A length-delimited field consists of: *

      - *
    • Tag (field number + wire type) encoded as a varint
      • - *
    • Length of the data encoded as a varint
      • - *
    • The actual data bytes
      • + *
    • Tag (field number + wire type) encoded as a varint
      • + *
    • Length of the data encoded as a varint
      • + *
    • The actual data bytes
      • *
    * * @param tag the protobuf field tag (field number << 3 | wire type) @@ -86,15 +86,15 @@ private static int calculateFieldSize(int tag, int dataLength) { /** * Calculates how many bytes a varint encoding will require for the given value. *

    - * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is - * a continuation flag). This means: + * Protobuf uses varint encoding where each byte stores 7 bits of data (the 8th bit is a continuation flag). This + * means: *

      - *
    • 1 byte: 0 to 127 (2^7 - 1)
      • - *
    • 2 bytes: 128 to 16,383 (2^14 - 1)
      • - *
    • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
      • - *
    • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
      • - *
    • 5 bytes: 268,435,456 to 4,294,967,295 (max unsigned 32-bit int)
      • - *
    • 10 bytes: negative numbers (sign-extended to 64 bits in varint encoding)
      • + *
    • 1 byte: 0 to 127 (2^7 - 1)
      • + *
    • 2 bytes: 128 to 16,383 (2^14 - 1)
      • + *
    • 3 bytes: 16,384 to 2,097,151 (2^21 - 1)
      • + *
    • 4 bytes: 2,097,152 to 268,435,455 (2^28 - 1)
      • + *
    • 5 bytes: 268,435,456 to 4,294,967,295 (max unsigned 32-bit int)
      • + *
    • 10 bytes: negative numbers (sign-extended to 64 bits in varint encoding)
      • *
    * * @param value the integer value to encode @@ -102,16 +102,16 @@ private static int calculateFieldSize(int tag, int dataLength) { */ private static int sizeOfVarint(int value) { if (value < 0) - return 10; // Negative numbers use sign extension, always 10 bytes - if (value < VARINT_BYTE_THRESHOLD) // 2^7 = 128 + return 10; // Negative numbers use sign extension, always 10 bytes + if (value < VARINT_BYTE_THRESHOLD) // 2^7 = 128 return 1; - if (value < 16384) // 2^14 + if (value < 16384) // 2^14 return 2; - if (value < 2097152) // 2^21 + if (value < 2097152) // 2^21 return 3; - if (value < 268435456) // 2^28 + if (value < 268435456) // 2^28 return 4; - return 5; // Max unsigned 32-bit int requires 5 bytes + return 5; // Max unsigned 32-bit int requires 5 bytes } /** @@ -119,9 +119,9 @@ private static int sizeOfVarint(int value) { *

    * A length-delimited field consists of: *

      - *
    • Tag (field number + wire type) encoded as a varint
      • - *
    • Length of the data encoded as a varint
      • - *
    • The actual data bytes
      • + *
    • Tag (field number + wire type) encoded as a varint
      • + *
    • Length of the data encoded as a varint
      • + *
    • The actual data bytes
      • *
    * * @param buffer the buffer to write to @@ -146,21 +146,21 @@ private static int writeField(Uint8Array buffer, int pos, int tag, Uint8Array da *

    * Varint encoding works by: *

      - *
    1. Taking the lowest 7 bits of the value
      2. - *
    2. Setting the 8th bit to 1 if more bytes follow (continuation flag)
      3. - *
    3. Writing the byte to the buffer
      4. - *
    4. Shifting the value right by 7 bits
      5. - *
    5. Repeating until the value is less than 128
      6. - *
    6. Writing the final byte without the continuation flag (8th bit = 0)
      7. + *
    7. Taking the lowest 7 bits of the value
      8. + *
    8. Setting the 8th bit to 1 if more bytes follow (continuation flag)
      9. + *
    9. Writing the byte to the buffer
      10. + *
    10. Shifting the value right by 7 bits
      11. + *
    11. Repeating until the value is less than 128
      12. + *
    12. Writing the final byte without the continuation flag (8th bit = 0)
      13. *
    *

    * Example: encoding 300 *

      - *
    • 300 in binary: 100101100
      • - *
    • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
      • - *
    • Shift: 300 >>> 7 = 2
      • - *
    • Second byte: 2 (no continuation flag)
      • - *
    • Result: [172, 2]
      • + *
    • 300 in binary: 100101100
      • + *
    • First byte: (300 & 0x7F) | 0x80 = 0b00101100 | 0b10000000 = 172 (0xAC)
      • + *
    • Shift: 300 >>> 7 = 2
      • + *
    • Second byte: 2 (no continuation flag)
      • + *
    • Result: [172, 2]
      • *
    * * @param buffer the buffer to write to @@ -173,7 +173,7 @@ private static int writeVarint(Uint8Array buffer, int pos, int value) { // Extract lowest 7 bits and set continuation flag (8th bit = 1) buffer.setAt(pos++, (double) ((value & VARINT_DATA_MASK) | VARINT_CONTINUATION_BIT)); // Shift right by 7 to process next chunk - value >>>= 7; // Unsigned right shift to handle large positive values + value >>>= 7; // Unsigned right shift to handle large positive values } // Write final byte (no continuation flag, 8th bit = 0) buffer.setAt(pos++, (double) value); diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index b706d895467..25ef1729a08 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.web.client.api.remotefilesource; @@ -49,10 +49,9 @@ *

    * Events: *

      - *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client. - * This event MUST have exactly one listener registered. Attempting to register more than one listener - * will throw an IllegalStateException. Receiving a resource request without a registered listener - * will also throw an IllegalStateException.
      • + *
    • {@link #EVENT_REQUEST_SOURCE}: Fired when the server requests a resource from the client. This event MUST have + * exactly one listener registered. Attempting to register more than one listener will throw an IllegalStateException. + * Receiving a resource request without a registered listener will also throw an IllegalStateException.
      • *
    */ @JsType(namespace = "dh.remotefilesource", name = "RemoteFileSourceService") @@ -123,8 +122,8 @@ private static Promise fetchPluginFlightInfo(WorkerConnection connec descriptor.setCmd(anyWrappedBytes); // Send the getFlightInfo request - return Callbacks.grpcUnaryPromise(c -> - connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); + return Callbacks.grpcUnaryPromise( + c -> connection.flightServiceClient().getFlightInfo(descriptor, connection.metadata(), c::apply)); } /** @@ -233,12 +232,10 @@ private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest mes /** - * Sets the execution context on the server to identify this message stream as active - * for script execution. + * Sets the execution context on the server to identify this message stream as active for script execution. * - * @param resourcePaths array of resource paths to resolve from remote source - * (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]), - * or null/empty for no specific resources + * @param resourcePaths array of resource paths to resolve from remote source (e.g., ["com/example/Test.groovy", + * "org/mycompany/Utils.groovy"]), or null/empty for no specific resources * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod @@ -265,7 +262,8 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) * @param requestId unique request ID * @return the constructed RemoteFileSourceClientRequest */ - private static @NotNull RemoteFileSourceClientRequest getSetExecutionContextRequest(String[] resourcePaths, String requestId) { + private static @NotNull RemoteFileSourceClientRequest getSetExecutionContextRequest(String[] resourcePaths, + String requestId) { SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); if (resourcePaths != null) { @@ -326,11 +324,11 @@ public String getResourceName() { * Responds to this resource request with the given content. * * @param content the resource content (String | Uint8Array | null): - *
      - *
    • String - will be UTF-8 encoded before sending to server
      • - *
    • Uint8Array - sent as-is to server
      • - *
    • null - indicates resource was not found
      • - *
    + *
      + *
    • String - will be UTF-8 encoded before sending to server
      • + *
    • Uint8Array - sent as-is to server
      • + *
    • null - indicates resource was not found
      • + *
    */ @JsMethod public void respond(@JsNullable ResourceContentUnion content) { diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java index 26135d2f1ba..8728839edd8 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/ResourceContentUnion.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.web.client.api.remotefilesource; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java index 8ba1dcc4846..09fc475322c 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; @@ -293,4 +293,4 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(); public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(boolean includeInstance); -} \ No newline at end of file +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java index eed1869744a..88b401232a6 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaRequest.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java index d06ac259b4a..057fd695300 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceMetaResponse.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java index b87d9dcb742..afc60285df6 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourcePluginFetchRequest.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java index f659e2336b2..308b783fc50 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index e8c069eb1ee..83f732e4ace 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java index 09d9d1861d8..6f9b83938c7 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextResponse.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java index a5a156433ed..00be8f422eb 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceclientrequest; @@ -12,4 +12,4 @@ namespace = JsPackage.GLOBAL) public class RequestCase { public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT; -} \ No newline at end of file +} diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java index 72cad9f6ff9..0a60aa50941 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java @@ -1,5 +1,5 @@ // -// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceserverrequest; From 58f591902d016f5d9af1b24e29b38f05a76f8de0 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Mon, 2 Mar 2026 18:15:39 -0600 Subject: [PATCH 72/78] Made execution context updating more targetted. (#DH-20578) --- .../engine/context/ExecutionContext.java | 15 ++++++++++ .../engine/util/GroovyDeephavenSession.java | 28 ++++++++++--------- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/engine/context/src/main/java/io/deephaven/engine/context/ExecutionContext.java b/engine/context/src/main/java/io/deephaven/engine/context/ExecutionContext.java index 58e24ff108f..f0794ef0abc 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/ExecutionContext.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/ExecutionContext.java @@ -187,6 +187,21 @@ public ExecutionContext withQueryScope(QueryScope queryScope) { operationInitializer); } + /** + * Returns, or creates, an execution context with the given value for {@code queryCompiler} and existing values for + * the other members. + * + * @param queryCompiler the query compiler to use instead + * @return the execution context + */ + public ExecutionContext withQueryCompiler(QueryCompiler queryCompiler) { + if (queryCompiler == this.queryCompiler) { + return this; + } + return new ExecutionContext(isSystemic, authContext, queryLibrary, queryScope, queryCompiler, updateGraph, + operationInitializer); + } + /** * Execute runnable within this execution context. diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 2a1beb45612..70a6f70fab3 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -132,6 +132,9 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE /** Contains imports to be applied to .groovy files loaded from the classpath */ private final ImportCustomizer loadedGroovyScriptImports; + private final CompilerConfiguration scriptConfig; + private final CompilerConfiguration consoleConfig; + private final Set dynamicClasses; private final Map bindingBackingMap; @@ -209,8 +212,8 @@ public static GroovyDeephavenSession of( return new GroovyDeephavenSession(updateGraph, operationInitializer, objectTypeLookup, changeListener, - runScripts, classCacheDirectory, consoleImports, loadedGroovyScriptImports, bindingBackingMap, - groovyShell); + runScripts, classCacheDirectory, consoleImports, loadedGroovyScriptImports, scriptConfig, + consoleConfig, bindingBackingMap, groovyShell); } private GroovyDeephavenSession( @@ -222,6 +225,8 @@ private GroovyDeephavenSession( final File classCacheDirectory, final ImportCustomizer consoleImports, final ImportCustomizer loadedGroovyScriptImports, + final CompilerConfiguration scriptConfig, + final CompilerConfiguration consoleConfig, final Map bindingBackingMap, final DeephavenGroovyShell groovyShell) throws IOException { @@ -232,6 +237,8 @@ private GroovyDeephavenSession( this.consoleImports = consoleImports; this.loadedGroovyScriptImports = loadedGroovyScriptImports; + this.scriptConfig = scriptConfig; + this.consoleConfig = consoleConfig; this.dynamicClasses = new HashSet<>(); this.bindingBackingMap = bindingBackingMap; @@ -349,20 +356,15 @@ private void refreshClassLoader() { deleteCachedClassFiles(classCacheDirectory); - // Create a fresh GroovyClassLoader - CompilerConfiguration config = new CompilerConfiguration(); - config.getCompilationCustomizers().add(consoleImports); - config.setTargetDirectory(classCacheDirectory); - GroovyClassLoader classLoader = new GroovyClassLoader(STATIC_LOADER, config); + // Create fresh classloader - reusing the existing configurations + GroovyClassLoader scriptClassLoader = new GroovyClassLoader(STATIC_LOADER, scriptConfig); - // Update the session's ExecutionContext with fresh QueryCompiler - executionContext = createExecutionContext( - executionContext.getUpdateGraph(), - executionContext.getOperationInitializer(), - classLoader); + // Update the execution context with a fresh QueryCompiler + final QueryCompiler freshCompiler = QueryCompilerImpl.create(classCacheDirectory, scriptClassLoader); + executionContext = executionContext.withQueryCompiler(freshCompiler); // Permanently replace groovyShell with fresh shell - groovyShell = new DeephavenGroovyShell(classLoader, groovyShell.getContext(), config); + groovyShell = new DeephavenGroovyShell(scriptClassLoader, groovyShell.getContext(), consoleConfig); } @Override From 57e158f886ee2afa30414aa9bcf60b24b7e90939 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 3 Mar 2026 10:07:19 -0600 Subject: [PATCH 73/78] Addressed code review comments (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 15 +++++++-------- .../RemoteFileSourcePlugin.java | 3 --- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 6fc2681cee7..9093d4c0b76 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -48,9 +48,13 @@ public class RemoteFileSourceMessageStream implements ObjectType.MessageStream, * provider with the RemoteFileSourceClassLoader. * * @param connection the message stream connection to the client + * @throws ObjectCommunicationException if the initial message cannot be sent to the client */ - public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) { + public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) + throws ObjectCommunicationException { this.connection = connection; + // Send initial empty message to client as required by the ObjectType contract + connection.onData(ByteBuffer.allocate(0)); // Register this instance as a provider with the RemoteFileSourceClassLoader registerWithClassLoader(); } @@ -74,12 +78,7 @@ public boolean canSourceResource(String resourceName) { return false; } - RemoteFileSourceExecutionContext context = executionContext; - - List resourcePaths = context.getResourcePaths(); - if (resourcePaths.isEmpty()) { - return false; - } + List resourcePaths = executionContext.getResourcePaths(); for (String contextResourcePath : resourcePaths) { if (resourceName.equals(contextResourcePath)) { @@ -104,7 +103,7 @@ public CompletableFuture requestResource(String resourceName) { if (!isActive()) { log.warn().append("Request for resource ").append(resourceName) .append(" on inactive message stream").endl(); - return CompletableFuture.completedFuture(null); + return CompletableFuture.failedFuture(new IllegalStateException("Inactive message stream")); } log.info().append("Requesting resource: ").append(resourceName).endl(); diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java index 06f8f7cff01..a106d0c1daf 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourcePlugin.java @@ -47,9 +47,6 @@ public MessageStream compatibleClientConnection(Object object, MessageStream con throw new ObjectCommunicationException("Expected RemoteFileSource marker object, got " + object.getClass()); } - // Send initial empty message to client as required by the ObjectType contract - connection.onData(ByteBuffer.allocate(0)); - // Return a new bidirectional message stream for this connection return new RemoteFileSourceMessageStream(connection); } From 38e5baebff1061e5040af2c4273e1a6a29c3955f Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Tue, 3 Mar 2026 11:00:28 -0600 Subject: [PATCH 74/78] Renamed RemoteFileSourceClientRequest -> RemoteFileSourceClientMessage and RemoteFileSourceServerRequest -> RemoteFileSourceServerMessage (#DH-20578) --- .../RemoteFileSourceMessageStream.java | 18 ++--- .../proto/remotefilesource.proto | 4 +- .../JsRemoteFileSourceService.java | 31 ++++----- ...ava => RemoteFileSourceClientMessage.java} | 68 +++++++++---------- ...ava => RemoteFileSourceServerMessage.java} | 48 ++++++------- .../RequestCase.java | 4 +- .../RequestCase.java | 4 +- 7 files changed, 88 insertions(+), 89 deletions(-) rename web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/{RemoteFileSourceClientRequest.java => RemoteFileSourceClientMessage.java} (79%) rename web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/{RemoteFileSourceServerRequest.java => RemoteFileSourceServerMessage.java} (75%) rename web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/{remotefilesourceclientrequest => remotefilesourceclientmessage}/RequestCase.java (80%) rename web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/{remotefilesourceserverrequest => remotefilesourceservermessage}/RequestCase.java (80%) diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 9093d4c0b76..85b505485d7 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -10,10 +10,10 @@ import io.deephaven.io.logger.Logger; import io.deephaven.plugin.type.ObjectCommunicationException; import io.deephaven.plugin.type.ObjectType; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceClientMessage; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; -import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerRequest; +import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerMessage; import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; @@ -119,9 +119,9 @@ public CompletableFuture requestResource(String resourceName) { .setResourceName(resourceName) .build(); - // Wrap in RemoteFileSourceServerRequest (server→client) - RemoteFileSourceServerRequest message = - RemoteFileSourceServerRequest.newBuilder() + // Wrap in RemoteFileSourceServerMessage (server→client) + RemoteFileSourceServerMessage message = + RemoteFileSourceServerMessage.newBuilder() .setRequestId(requestId) .setMetaRequest(metaRequest) .build(); @@ -207,7 +207,7 @@ public static void clearExecutionContext() { } /** - * Handles incoming data from the client. Parses RemoteFileSourceClientRequest messages and processes meta responses + * Handles incoming data from the client. Parses RemoteFileSourceClientMessage messages and processes meta responses * or execution context updates from the client. * * @param payload the message payload containing the protobuf data @@ -219,7 +219,7 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun try { byte[] bytes = new byte[payload.remaining()]; payload.get(bytes); - RemoteFileSourceClientRequest message = RemoteFileSourceClientRequest.parseFrom(bytes); + RemoteFileSourceClientMessage message = RemoteFileSourceClientMessage.parseFrom(bytes); if (message.hasMetaResponse()) { handleMetaResponse(message.getRequestId(), message.getMetaResponse()); @@ -231,7 +231,7 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun throw new ObjectCommunicationException("Received unknown message type from client"); } } catch (InvalidProtocolBufferException e) { - log.error().append("Failed to parse RemoteFileSourceClientRequest: ").append(e).endl(); + log.error().append("Failed to parse RemoteFileSourceClientMessage: ").append(e).endl(); throw new ObjectCommunicationException("Failed to parse message", e); } } @@ -286,7 +286,7 @@ private void sendExecutionContextAcknowledgment(String requestId) { .setSuccess(true) .build(); - RemoteFileSourceServerRequest serverRequest = RemoteFileSourceServerRequest.newBuilder() + RemoteFileSourceServerMessage serverRequest = RemoteFileSourceServerMessage.newBuilder() .setRequestId(requestId) .setSetExecutionContextResponse(response) .build(); diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 20cd589fce1..36206c8e9be 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -12,7 +12,7 @@ option go_package = "github.com/deephaven/deephaven-core/go/internal/proto/remot import "deephaven_core/proto/ticket.proto"; // Server → Client: Requests sent from server to client via MessageStream -message RemoteFileSourceServerRequest { +message RemoteFileSourceServerMessage { // Unique identifier for this request, used to correlate the response string request_id = 1; @@ -26,7 +26,7 @@ message RemoteFileSourceServerRequest { } // Client → Server: Requests/responses sent from client to server via MessageStream -message RemoteFileSourceClientRequest { +message RemoteFileSourceClientMessage { // The request_id from the ServerRequest this is responding to (if applicable) string request_id = 1; diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index 25ef1729a08..f7434e0f5bc 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -6,16 +6,15 @@ import com.vertispan.tsdefs.annotations.TsInterface; import com.vertispan.tsdefs.annotations.TsName; import elemental2.core.Uint8Array; -import elemental2.dom.DomGlobal; import elemental2.dom.TextEncoder; import elemental2.promise.Promise; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightDescriptor; import io.deephaven.javascript.proto.dhinternal.arrow.flight.protocol.flight_pb.FlightInfo; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientMessage; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceMetaResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourcePluginFetchRequest; -import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest; +import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerMessage; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextRequest; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.SetExecutionContextResponse; import io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.ticket_pb.Ticket; @@ -183,9 +182,9 @@ private Promise connect() { private void handleMessage(Event event) { Uint8Array payload = event.getDetail().getDataAsU8(); - RemoteFileSourceServerRequest message; + RemoteFileSourceServerMessage message; try { - message = RemoteFileSourceServerRequest.deserializeBinary(payload); + message = RemoteFileSourceServerMessage.deserializeBinary(payload); } catch (Exception e) { // Failed to parse as proto throw new IllegalStateException("Received unparseable message from server", e); @@ -206,7 +205,7 @@ private void handleMessage(Event event) { * * @param message the server request message */ - private void handleMetaRequest(RemoteFileSourceServerRequest message) { + private void handleMetaRequest(RemoteFileSourceServerMessage message) { if (!hasListeners(EVENT_REQUEST_SOURCE)) { throw new IllegalStateException( "Received resource request from server but no listener is registered for EVENT_REQUEST_SOURCE. " @@ -221,7 +220,7 @@ private void handleMetaRequest(RemoteFileSourceServerRequest message) { * * @param message the server request message */ - private void handleSetExecutionContextResponse(RemoteFileSourceServerRequest message) { + private void handleSetExecutionContextResponse(RemoteFileSourceServerMessage message) { String requestId = message.getRequestId(); LazyPromise promise = pendingSetExecutionContextRequests.remove(requestId); if (promise != null) { @@ -248,7 +247,7 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) pendingSetExecutionContextRequests.put(requestId, promise); // Send the request - RemoteFileSourceClientRequest clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); + RemoteFileSourceClientMessage clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); sendClientRequest(clientRequest); // Return a promise with built-in timeout @@ -256,13 +255,13 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) } /** - * Helper method to build a RemoteFileSourceClientRequest for setting execution context. + * Helper method to build a RemoteFileSourceClientMessage for setting execution context. * * @param resourcePaths array of resource paths to resolve * @param requestId unique request ID - * @return the constructed RemoteFileSourceClientRequest + * @return the constructed RemoteFileSourceClientMessage */ - private static @NotNull RemoteFileSourceClientRequest getSetExecutionContextRequest(String[] resourcePaths, + private static @NotNull RemoteFileSourceClientMessage getSetExecutionContextRequest(String[] resourcePaths, String requestId) { SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); @@ -270,18 +269,18 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) setContextRequest.setResourcePathsList(resourcePaths); } - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + RemoteFileSourceClientMessage clientRequest = new RemoteFileSourceClientMessage(); clientRequest.setRequestId(requestId); clientRequest.setSetExecutionContext(setContextRequest); return clientRequest; } /** - * Helper method to send a RemoteFileSourceClientRequest to the server. + * Helper method to send a RemoteFileSourceClientMessage to the server. * * @param clientRequest the client request to send */ - private void sendClientRequest(RemoteFileSourceClientRequest clientRequest) { + private void sendClientRequest(RemoteFileSourceClientMessage clientRequest) { // Serialize the protobuf message to bytes Uint8Array messageBytes = clientRequest.serializeBinary(); @@ -353,8 +352,8 @@ public void respond(@JsNullable ResourceContentUnion content) { } } - // Wrap in RemoteFileSourceClientRequest (client→server) - RemoteFileSourceClientRequest clientRequest = new RemoteFileSourceClientRequest(); + // Wrap in RemoteFileSourceClientMessage (client→server) + RemoteFileSourceClientMessage clientRequest = new RemoteFileSourceClientMessage(); clientRequest.setRequestId(requestId); clientRequest.setMetaResponse(response); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java similarity index 79% rename from web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java rename to web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java index 09fc475322c..b0aad6fb63b 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java @@ -14,9 +14,9 @@ @JsType( isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest", + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientMessage", namespace = JsPackage.GLOBAL) -public class RemoteFileSourceClientRequest { +public class RemoteFileSourceClientMessage { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface ToObjectReturnType { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -24,7 +24,7 @@ public interface MetaResponseFieldType { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface GetContentUnionType { @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType of( + static RemoteFileSourceClientMessage.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType of( Object o) { return Js.cast(o); } @@ -51,12 +51,12 @@ default boolean isUint8Array() { } @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType create() { + static RemoteFileSourceClientMessage.ToObjectReturnType.MetaResponseFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @JsProperty - RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType getContent(); + RemoteFileSourceClientMessage.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType getContent(); @JsProperty String getError(); @@ -66,19 +66,19 @@ static RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType cr @JsProperty void setContent( - RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType content); + RemoteFileSourceClientMessage.ToObjectReturnType.MetaResponseFieldType.GetContentUnionType content); @JsOverlay default void setContent(String content) { setContent( - Js.uncheckedCast( + Js.uncheckedCast( content)); } @JsOverlay default void setContent(Uint8Array content) { setContent( - Js.uncheckedCast( + Js.uncheckedCast( content)); } @@ -92,7 +92,7 @@ default void setContent(Uint8Array content) { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface SetExecutionContextFieldType { @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType create() { + static RemoteFileSourceClientMessage.ToObjectReturnType.SetExecutionContextFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @@ -109,29 +109,29 @@ default void setResourcePathsList(String[] resourcePathsList) { } @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType create() { + static RemoteFileSourceClientMessage.ToObjectReturnType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @JsProperty - RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType getMetaResponse(); + RemoteFileSourceClientMessage.ToObjectReturnType.MetaResponseFieldType getMetaResponse(); @JsProperty String getRequestId(); @JsProperty - RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType getSetExecutionContext(); + RemoteFileSourceClientMessage.ToObjectReturnType.SetExecutionContextFieldType getSetExecutionContext(); @JsProperty void setMetaResponse( - RemoteFileSourceClientRequest.ToObjectReturnType.MetaResponseFieldType metaResponse); + RemoteFileSourceClientMessage.ToObjectReturnType.MetaResponseFieldType metaResponse); @JsProperty void setRequestId(String requestId); @JsProperty void setSetExecutionContext( - RemoteFileSourceClientRequest.ToObjectReturnType.SetExecutionContextFieldType setExecutionContext); + RemoteFileSourceClientMessage.ToObjectReturnType.SetExecutionContextFieldType setExecutionContext); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -141,7 +141,7 @@ public interface MetaResponseFieldType { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface GetContentUnionType { @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType of( + static RemoteFileSourceClientMessage.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType of( Object o) { return Js.cast(o); } @@ -168,12 +168,12 @@ default boolean isUint8Array() { } @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType create() { + static RemoteFileSourceClientMessage.ToObjectReturnType0.MetaResponseFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @JsProperty - RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType getContent(); + RemoteFileSourceClientMessage.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType getContent(); @JsProperty String getError(); @@ -183,19 +183,19 @@ static RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType c @JsProperty void setContent( - RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType content); + RemoteFileSourceClientMessage.ToObjectReturnType0.MetaResponseFieldType.GetContentUnionType content); @JsOverlay default void setContent(String content) { setContent( - Js.uncheckedCast( + Js.uncheckedCast( content)); } @JsOverlay default void setContent(Uint8Array content) { setContent( - Js.uncheckedCast( + Js.uncheckedCast( content)); } @@ -209,7 +209,7 @@ default void setContent(Uint8Array content) { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface SetExecutionContextFieldType { @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType create() { + static RemoteFileSourceClientMessage.ToObjectReturnType0.SetExecutionContextFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @@ -226,41 +226,41 @@ default void setResourcePathsList(String[] resourcePathsList) { } @JsOverlay - static RemoteFileSourceClientRequest.ToObjectReturnType0 create() { + static RemoteFileSourceClientMessage.ToObjectReturnType0 create() { return Js.uncheckedCast(JsPropertyMap.of()); } @JsProperty - RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType getMetaResponse(); + RemoteFileSourceClientMessage.ToObjectReturnType0.MetaResponseFieldType getMetaResponse(); @JsProperty String getRequestId(); @JsProperty - RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType getSetExecutionContext(); + RemoteFileSourceClientMessage.ToObjectReturnType0.SetExecutionContextFieldType getSetExecutionContext(); @JsProperty void setMetaResponse( - RemoteFileSourceClientRequest.ToObjectReturnType0.MetaResponseFieldType metaResponse); + RemoteFileSourceClientMessage.ToObjectReturnType0.MetaResponseFieldType metaResponse); @JsProperty void setRequestId(String requestId); @JsProperty void setSetExecutionContext( - RemoteFileSourceClientRequest.ToObjectReturnType0.SetExecutionContextFieldType setExecutionContext); + RemoteFileSourceClientMessage.ToObjectReturnType0.SetExecutionContextFieldType setExecutionContext); } - public static native RemoteFileSourceClientRequest deserializeBinary(Uint8Array bytes); + public static native RemoteFileSourceClientMessage deserializeBinary(Uint8Array bytes); - public static native RemoteFileSourceClientRequest deserializeBinaryFromReader( - RemoteFileSourceClientRequest message, Object reader); + public static native RemoteFileSourceClientMessage deserializeBinaryFromReader( + RemoteFileSourceClientMessage message, Object reader); public static native void serializeBinaryToWriter( - RemoteFileSourceClientRequest message, Object writer); + RemoteFileSourceClientMessage message, Object writer); - public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( - boolean includeInstance, RemoteFileSourceClientRequest msg); + public static native RemoteFileSourceClientMessage.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceClientMessage msg); public native void clearMetaResponse(); @@ -290,7 +290,7 @@ public static native RemoteFileSourceClientRequest.ToObjectReturnType toObject( public native void setSetExecutionContext(SetExecutionContextRequest value); - public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(); + public native RemoteFileSourceClientMessage.ToObjectReturnType0 toObject(); - public native RemoteFileSourceClientRequest.ToObjectReturnType0 toObject(boolean includeInstance); + public native RemoteFileSourceClientMessage.ToObjectReturnType0 toObject(boolean includeInstance); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerMessage.java similarity index 75% rename from web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java rename to web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerMessage.java index 308b783fc50..9069c393d6d 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceServerMessage.java @@ -13,15 +13,15 @@ @JsType( isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest", + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerMessage", namespace = JsPackage.GLOBAL) -public class RemoteFileSourceServerRequest { +public class RemoteFileSourceServerMessage { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface ToObjectReturnType { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface MetaRequestFieldType { @JsOverlay - static RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType create() { + static RemoteFileSourceServerMessage.ToObjectReturnType.MetaRequestFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @@ -35,7 +35,7 @@ static RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType cre @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface SetExecutionContextResponseFieldType { @JsOverlay - static RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType create() { + static RemoteFileSourceServerMessage.ToObjectReturnType.SetExecutionContextResponseFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @@ -47,29 +47,29 @@ static RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextRespo } @JsOverlay - static RemoteFileSourceServerRequest.ToObjectReturnType create() { + static RemoteFileSourceServerMessage.ToObjectReturnType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @JsProperty - RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType getMetaRequest(); + RemoteFileSourceServerMessage.ToObjectReturnType.MetaRequestFieldType getMetaRequest(); @JsProperty String getRequestId(); @JsProperty - RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); + RemoteFileSourceServerMessage.ToObjectReturnType.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); @JsProperty void setMetaRequest( - RemoteFileSourceServerRequest.ToObjectReturnType.MetaRequestFieldType metaRequest); + RemoteFileSourceServerMessage.ToObjectReturnType.MetaRequestFieldType metaRequest); @JsProperty void setRequestId(String requestId); @JsProperty void setSetExecutionContextResponse( - RemoteFileSourceServerRequest.ToObjectReturnType.SetExecutionContextResponseFieldType setExecutionContextResponse); + RemoteFileSourceServerMessage.ToObjectReturnType.SetExecutionContextResponseFieldType setExecutionContextResponse); } @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) @@ -77,7 +77,7 @@ public interface ToObjectReturnType0 { @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface MetaRequestFieldType { @JsOverlay - static RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType create() { + static RemoteFileSourceServerMessage.ToObjectReturnType0.MetaRequestFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @@ -91,7 +91,7 @@ static RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType cr @JsType(isNative = true, name = "?", namespace = JsPackage.GLOBAL) public interface SetExecutionContextResponseFieldType { @JsOverlay - static RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType create() { + static RemoteFileSourceServerMessage.ToObjectReturnType0.SetExecutionContextResponseFieldType create() { return Js.uncheckedCast(JsPropertyMap.of()); } @@ -103,41 +103,41 @@ static RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResp } @JsOverlay - static RemoteFileSourceServerRequest.ToObjectReturnType0 create() { + static RemoteFileSourceServerMessage.ToObjectReturnType0 create() { return Js.uncheckedCast(JsPropertyMap.of()); } @JsProperty - RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType getMetaRequest(); + RemoteFileSourceServerMessage.ToObjectReturnType0.MetaRequestFieldType getMetaRequest(); @JsProperty String getRequestId(); @JsProperty - RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); + RemoteFileSourceServerMessage.ToObjectReturnType0.SetExecutionContextResponseFieldType getSetExecutionContextResponse(); @JsProperty void setMetaRequest( - RemoteFileSourceServerRequest.ToObjectReturnType0.MetaRequestFieldType metaRequest); + RemoteFileSourceServerMessage.ToObjectReturnType0.MetaRequestFieldType metaRequest); @JsProperty void setRequestId(String requestId); @JsProperty void setSetExecutionContextResponse( - RemoteFileSourceServerRequest.ToObjectReturnType0.SetExecutionContextResponseFieldType setExecutionContextResponse); + RemoteFileSourceServerMessage.ToObjectReturnType0.SetExecutionContextResponseFieldType setExecutionContextResponse); } - public static native RemoteFileSourceServerRequest deserializeBinary(Uint8Array bytes); + public static native RemoteFileSourceServerMessage deserializeBinary(Uint8Array bytes); - public static native RemoteFileSourceServerRequest deserializeBinaryFromReader( - RemoteFileSourceServerRequest message, Object reader); + public static native RemoteFileSourceServerMessage deserializeBinaryFromReader( + RemoteFileSourceServerMessage message, Object reader); public static native void serializeBinaryToWriter( - RemoteFileSourceServerRequest message, Object writer); + RemoteFileSourceServerMessage message, Object writer); - public static native RemoteFileSourceServerRequest.ToObjectReturnType toObject( - boolean includeInstance, RemoteFileSourceServerRequest msg); + public static native RemoteFileSourceServerMessage.ToObjectReturnType toObject( + boolean includeInstance, RemoteFileSourceServerMessage msg); public native void clearMetaRequest(); @@ -167,7 +167,7 @@ public static native RemoteFileSourceServerRequest.ToObjectReturnType toObject( public native void setSetExecutionContextResponse(SetExecutionContextResponse value); - public native RemoteFileSourceServerRequest.ToObjectReturnType0 toObject(); + public native RemoteFileSourceServerMessage.ToObjectReturnType0 toObject(); - public native RemoteFileSourceServerRequest.ToObjectReturnType0 toObject(boolean includeInstance); + public native RemoteFileSourceServerMessage.ToObjectReturnType0 toObject(boolean includeInstance); } diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientmessage/RequestCase.java similarity index 80% rename from web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java rename to web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientmessage/RequestCase.java index 00be8f422eb..402849f4f93 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceclientmessage/RequestCase.java @@ -1,14 +1,14 @@ // // Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceclientrequest; +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceclientmessage; import jsinterop.annotations.JsPackage; import jsinterop.annotations.JsType; @JsType( isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientRequest.RequestCase", + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceClientMessage.RequestCase", namespace = JsPackage.GLOBAL) public class RequestCase { public static int META_RESPONSE, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT; diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceservermessage/RequestCase.java similarity index 80% rename from web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java rename to web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceservermessage/RequestCase.java index 0a60aa50941..5d3752b8728 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceserverrequest/RequestCase.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/remotefilesourceservermessage/RequestCase.java @@ -1,14 +1,14 @@ // // Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // -package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceserverrequest; +package io.deephaven.javascript.proto.dhinternal.io.deephaven_core.proto.remotefilesource_pb.remotefilesourceservermessage; import jsinterop.annotations.JsPackage; import jsinterop.annotations.JsType; @JsType( isNative = true, - name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerRequest.RequestCase", + name = "dhinternal.io.deephaven_core.proto.remotefilesource_pb.RemoteFileSourceServerMessage.RequestCase", namespace = JsPackage.GLOBAL) public class RequestCase { public static int META_REQUEST, REQUEST_NOT_SET, SET_EXECUTION_CONTEXT_RESPONSE; From c522e5d6b501521adfbe82efb05c03bbc23bcaf5 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Thu, 5 Mar 2026 09:25:43 -0600 Subject: [PATCH 75/78] Added is_dirty flag to execution context (#DH-20578) --- .../util/RemoteFileSourceClassLoader.java | 15 +++++ .../engine/util/RemoteFileSourceProvider.java | 8 +++ .../engine/util/GroovyDeephavenSession.java | 15 +++-- .../RemoteFileSourceMessageStream.java | 61 +++++++++++++------ .../proto/remotefilesource.proto | 3 + 5 files changed, 77 insertions(+), 25 deletions(-) diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java index 75121a76ae5..fb8784892c5 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceClassLoader.java @@ -110,6 +110,21 @@ public boolean hasConfiguredRemoteSources() { return false; } + /** + * Returns whether the current execution context is dirty, indicating that remote sources have changed and the cache + * should be cleared. This method is used by GroovyDeephavenSession to determine when to refresh the class cache. + * + * @return true if there is a dirty execution context, false otherwise + */ + public boolean isDirty() { + for (RemoteFileSourceProvider candidate : providers) { + if (candidate.isActive() && candidate.isDirty()) { + return true; + } + } + return false; + } + /** * Gets the resource with the specified name by checking registered providers. * diff --git a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java index ebcb6a3310d..dae0ee077f9 100644 --- a/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java +++ b/engine/base/src/main/java/io/deephaven/engine/util/RemoteFileSourceProvider.java @@ -33,6 +33,14 @@ public interface RemoteFileSourceProvider { */ boolean hasConfiguredResources(); + /** + * Check if this provider's execution context is dirty, indicating that remote sources have changed and the cache + * should be cleared. + * + * @return true if this provider is active and dirty, false otherwise + */ + boolean isDirty(); + /** * Request a resource from the remote source. * diff --git a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java index 70a6f70fab3..b07567a7ad2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/GroovyDeephavenSession.java @@ -381,12 +381,15 @@ protected void evaluate(String command, String scriptName) { final RemoteFileSourceClassLoader remoteLoader = RemoteFileSourceClassLoader.getInstance(); final boolean hasRemoteSources = remoteLoader.hasConfiguredRemoteSources(); - - // If previous run had remote sources enabled, we need to clear the cache in case anything has changed. - // If current run has remote sources enabled, we also need to clear the cache so that cached local classes have - // opportunity to be remotely fetched. If remote sources aren't involved, no need to clear the cache. - if (previousEvalHadRemoteSources || hasRemoteSources) { - log.debug("Remote file sourcing enabled or was previously enabled. Clearing class cache."); + final boolean isDirty = remoteLoader.isDirty(); + + // Clear the cache in two cases: + // 1. is_dirty flag is set - remote sources have changed + // 2. Previous eval had remote sources but current does not - catches edge case where script is run + // without providing execution context at all, and we need to clear from previous remote source scenario + if (isDirty || (previousEvalHadRemoteSources && !hasRemoteSources)) { + log.debug("Clearing class cache. isDirty: " + isDirty + ", previousEvalHadRemoteSources: " + + previousEvalHadRemoteSources + ", hasRemoteSources: " + hasRemoteSources); refreshClassLoader(); } diff --git a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java index 85b505485d7..a61b89834d2 100644 --- a/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java +++ b/plugin/remotefilesource/src/main/java/io.deephaven.remotefilesource/RemoteFileSourceMessageStream.java @@ -14,6 +14,7 @@ import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest; import io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse; import io.deephaven.proto.backplane.grpc.RemoteFileSourceServerMessage; +import io.deephaven.proto.backplane.grpc.SetExecutionContextRequest; import io.deephaven.proto.backplane.grpc.SetExecutionContextResponse; import java.nio.ByteBuffer; @@ -68,7 +69,6 @@ public RemoteFileSourceMessageStream(final ObjectType.MessageStream connection) */ @Override public boolean canSourceResource(String resourceName) { - // Only active if this instance is the currently active message stream if (!isActive()) { return false; } @@ -78,9 +78,7 @@ public boolean canSourceResource(String resourceName) { return false; } - List resourcePaths = executionContext.getResourcePaths(); - - for (String contextResourcePath : resourcePaths) { + for (String contextResourcePath : executionContext.getResourcePaths()) { if (resourceName.equals(contextResourcePath)) { log.debug().append("Can source: ").append(resourceName).endl(); return true; @@ -159,11 +157,17 @@ public boolean isActive() { */ @Override public boolean hasConfiguredResources() { - if (!isActive()) { - return false; - } - RemoteFileSourceExecutionContext context = executionContext; - return context != null && !context.getResourcePaths().isEmpty(); + return isActive() && !executionContext.getResourcePaths().isEmpty(); + } + + /** + * Checks if this provider's execution context is dirty. + * + * @return true if this provider is active and the execution context is dirty, false otherwise + */ + @Override + public boolean isDirty() { + return isActive() && executionContext.isDirty(); } /** @@ -184,16 +188,19 @@ public boolean hasConfiguredResources() { * * @param messageStream the message stream to set as active (must not be null) * @param resourcePaths list of resource paths (e.g., "package/MyScript.groovy") to resolve from remote source + * @param isDirty whether remote sources have changed and cache should be cleared * @throws IllegalArgumentException if messageStream is null */ - public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, List resourcePaths) { + public static void setExecutionContext(RemoteFileSourceMessageStream messageStream, List resourcePaths, + boolean isDirty) { if (messageStream == null) { throw new IllegalArgumentException("messageStream must not be null"); } - executionContext = new RemoteFileSourceExecutionContext(messageStream, resourcePaths); + executionContext = new RemoteFileSourceExecutionContext(messageStream, resourcePaths, isDirty); log.info().append("Set execution context with ") - .append(executionContext.getResourcePaths().size()).append(" resource paths").endl(); + .append(executionContext.getResourcePaths().size()).append(" resource paths") + .append(", isDirty: ").append(isDirty).endl(); } /** @@ -206,6 +213,7 @@ public static void clearExecutionContext() { } } + /** * Handles incoming data from the client. Parses RemoteFileSourceClientMessage messages and processes meta responses * or execution context updates from the client. @@ -224,8 +232,7 @@ public void onData(ByteBuffer payload, Object... references) throws ObjectCommun if (message.hasMetaResponse()) { handleMetaResponse(message.getRequestId(), message.getMetaResponse()); } else if (message.hasSetExecutionContext()) { - handleSetExecutionContext(message.getRequestId(), - message.getSetExecutionContext().getResourcePathsList()); + handleSetExecutionContext(message.getRequestId(), message.getSetExecutionContext()); } else { log.error().append("Received unknown message type from client").endl(); throw new ObjectCommunicationException("Received unknown message type from client"); @@ -266,12 +273,16 @@ private void handleMetaResponse(String requestId, RemoteFileSourceMetaResponse r * Handles a request from the client to set the execution context. * * @param requestId the request ID - * @param resourcePaths the list of resource paths to resolve from remote source + * @param setExecutionContext the SetExecutionContextRequest containing resource paths and isDirty flag */ - private void handleSetExecutionContext(String requestId, List resourcePaths) { - setExecutionContext(this, resourcePaths); + private void handleSetExecutionContext(String requestId, SetExecutionContextRequest setExecutionContext) { + boolean isDirty = setExecutionContext.getIsDirty(); + List resourcePaths = setExecutionContext.getResourcePathsList(); + + setExecutionContext(this, resourcePaths, isDirty); log.info().append("Client set execution context for this message stream with ") - .append(resourcePaths.size()).append(" resource paths").endl(); + .append(resourcePaths.size()).append(" resource paths") + .append(", isDirty: ").append(isDirty).endl(); sendExecutionContextAcknowledgment(requestId); } @@ -346,17 +357,20 @@ private void unregisterFromClassLoader() { public static class RemoteFileSourceExecutionContext { private final RemoteFileSourceMessageStream activeMessageStream; private final List resourcePaths; + private final boolean isDirty; /** * Creates a new execution context. * * @param activeMessageStream the active message stream * @param resourcePaths list of resource paths to resolve from remote source + * @param isDirty whether remote sources have changed and cache should be cleared */ public RemoteFileSourceExecutionContext(RemoteFileSourceMessageStream activeMessageStream, - List resourcePaths) { + List resourcePaths, boolean isDirty) { this.activeMessageStream = activeMessageStream; this.resourcePaths = resourcePaths; + this.isDirty = isDirty; } /** @@ -376,6 +390,15 @@ public RemoteFileSourceMessageStream getActiveMessageStream() { public List getResourcePaths() { return resourcePaths; } + + /** + * Gets whether remote sources have changed and cache should be cleared. + * + * @return true if dirty, false otherwise + */ + public boolean isDirty() { + return isDirty; + } } } diff --git a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto index 36206c8e9be..7560ba56648 100644 --- a/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto +++ b/proto/proto-backplane-grpc/src/main/proto/deephaven_core/proto/remotefilesource.proto @@ -62,6 +62,9 @@ message SetExecutionContextRequest { // Resource paths that should be resolved from the remote source // (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) repeated string resource_paths = 1; + + // Indicates whether remote sources have changed and cache should be cleared + bool is_dirty = 2; } // Response acknowledging execution context was set From ae4119657ffe18648d06e567cb79231444532cfd Mon Sep 17 00:00:00 2001 From: Colin Alworth Date: Thu, 5 Mar 2026 14:32:41 -0600 Subject: [PATCH 76/78] regen gwt bindings --- .../RemoteFileSourceClientMessage.java | 12 ++++++++++++ .../SetExecutionContextRequest.java | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java index b0aad6fb63b..fa1377115b8 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/RemoteFileSourceClientMessage.java @@ -99,6 +99,12 @@ static RemoteFileSourceClientMessage.ToObjectReturnType.SetExecutionContextField @JsProperty JsArray getResourcePathsList(); + @JsProperty + boolean isIsDirty(); + + @JsProperty + void setIsDirty(boolean isDirty); + @JsProperty void setResourcePathsList(JsArray resourcePathsList); @@ -216,6 +222,12 @@ static RemoteFileSourceClientMessage.ToObjectReturnType0.SetExecutionContextFiel @JsProperty JsArray getResourcePathsList(); + @JsProperty + boolean isIsDirty(); + + @JsProperty + void setIsDirty(boolean isDirty); + @JsProperty void setResourcePathsList(JsArray resourcePathsList); diff --git a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java index 83f732e4ace..728323a187a 100644 --- a/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java +++ b/web/client-backplane/src/main/java/io/deephaven/javascript/proto/dhinternal/io/deephaven_core/proto/remotefilesource_pb/SetExecutionContextRequest.java @@ -27,6 +27,12 @@ static SetExecutionContextRequest.ToObjectReturnType create() { @JsProperty JsArray getResourcePathsList(); + @JsProperty + boolean isIsDirty(); + + @JsProperty + void setIsDirty(boolean isDirty); + @JsProperty void setResourcePathsList(JsArray resourcePathsList); @@ -46,6 +52,12 @@ static SetExecutionContextRequest.ToObjectReturnType0 create() { @JsProperty JsArray getResourcePathsList(); + @JsProperty + boolean isIsDirty(); + + @JsProperty + void setIsDirty(boolean isDirty); + @JsProperty void setResourcePathsList(JsArray resourcePathsList); @@ -72,10 +84,14 @@ public static native SetExecutionContextRequest.ToObjectReturnType toObject( public native void clearResourcePathsList(); + public native boolean getIsDirty(); + public native JsArray getResourcePathsList(); public native Uint8Array serializeBinary(); + public native void setIsDirty(boolean value); + public native void setResourcePathsList(JsArray value); @JsOverlay From 38e71898ca804a2842c07bdcab7423bae19cf085 Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Wed, 11 Mar 2026 08:44:08 -0500 Subject: [PATCH 77/78] Ran ./gradlew updateProtobuf (#DH-20578) --- .../remotefilesource/remotefilesource.pb.go | 731 ++++++++++++++++++ .../proto/remotefilesource_pb2.py | 50 ++ .../proto/remotefilesource_pb2.pyi | 211 +++++ .../proto/remotefilesource_pb2_grpc.py | 24 + 4 files changed, 1016 insertions(+) create mode 100644 go/internal/proto/remotefilesource/remotefilesource.pb.go create mode 100644 py/client/deephaven_core/proto/remotefilesource_pb2.py create mode 100644 py/client/deephaven_core/proto/remotefilesource_pb2.pyi create mode 100644 py/client/deephaven_core/proto/remotefilesource_pb2_grpc.py diff --git a/go/internal/proto/remotefilesource/remotefilesource.pb.go b/go/internal/proto/remotefilesource/remotefilesource.pb.go new file mode 100644 index 00000000000..619507922bf --- /dev/null +++ b/go/internal/proto/remotefilesource/remotefilesource.pb.go @@ -0,0 +1,731 @@ +// +// Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v6.33.2 +// source: deephaven_core/proto/remotefilesource.proto + +package remotefilesource + +import ( + ticket "github.com/deephaven/deephaven-core/go/internal/proto/ticket" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Server → Client: Requests sent from server to client via MessageStream +type RemoteFileSourceServerMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Unique identifier for this request, used to correlate the response + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // Types that are assignable to Request: + // *RemoteFileSourceServerMessage_MetaRequest + // *RemoteFileSourceServerMessage_SetExecutionContextResponse + Request isRemoteFileSourceServerMessage_Request `protobuf_oneof:"request"` +} + +func (x *RemoteFileSourceServerMessage) Reset() { + *x = RemoteFileSourceServerMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoteFileSourceServerMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoteFileSourceServerMessage) ProtoMessage() {} + +func (x *RemoteFileSourceServerMessage) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoteFileSourceServerMessage.ProtoReflect.Descriptor instead. +func (*RemoteFileSourceServerMessage) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{0} +} + +func (x *RemoteFileSourceServerMessage) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (m *RemoteFileSourceServerMessage) GetRequest() isRemoteFileSourceServerMessage_Request { + if m != nil { + return m.Request + } + return nil +} + +func (x *RemoteFileSourceServerMessage) GetMetaRequest() *RemoteFileSourceMetaRequest { + if x, ok := x.GetRequest().(*RemoteFileSourceServerMessage_MetaRequest); ok { + return x.MetaRequest + } + return nil +} + +func (x *RemoteFileSourceServerMessage) GetSetExecutionContextResponse() *SetExecutionContextResponse { + if x, ok := x.GetRequest().(*RemoteFileSourceServerMessage_SetExecutionContextResponse); ok { + return x.SetExecutionContextResponse + } + return nil +} + +type isRemoteFileSourceServerMessage_Request interface { + isRemoteFileSourceServerMessage_Request() +} + +type RemoteFileSourceServerMessage_MetaRequest struct { + // Request source data/resource from the client + MetaRequest *RemoteFileSourceMetaRequest `protobuf:"bytes,2,opt,name=meta_request,json=metaRequest,proto3,oneof"` +} + +type RemoteFileSourceServerMessage_SetExecutionContextResponse struct { + // Acknowledgment that execution context was set + SetExecutionContextResponse *SetExecutionContextResponse `protobuf:"bytes,3,opt,name=set_execution_context_response,json=setExecutionContextResponse,proto3,oneof"` +} + +func (*RemoteFileSourceServerMessage_MetaRequest) isRemoteFileSourceServerMessage_Request() {} + +func (*RemoteFileSourceServerMessage_SetExecutionContextResponse) isRemoteFileSourceServerMessage_Request() { +} + +// Client → Server: Requests/responses sent from client to server via MessageStream +type RemoteFileSourceClientMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The request_id from the ServerRequest this is responding to (if applicable) + RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // Types that are assignable to Request: + // *RemoteFileSourceClientMessage_MetaResponse + // *RemoteFileSourceClientMessage_SetExecutionContext + Request isRemoteFileSourceClientMessage_Request `protobuf_oneof:"request"` +} + +func (x *RemoteFileSourceClientMessage) Reset() { + *x = RemoteFileSourceClientMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoteFileSourceClientMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoteFileSourceClientMessage) ProtoMessage() {} + +func (x *RemoteFileSourceClientMessage) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoteFileSourceClientMessage.ProtoReflect.Descriptor instead. +func (*RemoteFileSourceClientMessage) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{1} +} + +func (x *RemoteFileSourceClientMessage) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +func (m *RemoteFileSourceClientMessage) GetRequest() isRemoteFileSourceClientMessage_Request { + if m != nil { + return m.Request + } + return nil +} + +func (x *RemoteFileSourceClientMessage) GetMetaResponse() *RemoteFileSourceMetaResponse { + if x, ok := x.GetRequest().(*RemoteFileSourceClientMessage_MetaResponse); ok { + return x.MetaResponse + } + return nil +} + +func (x *RemoteFileSourceClientMessage) GetSetExecutionContext() *SetExecutionContextRequest { + if x, ok := x.GetRequest().(*RemoteFileSourceClientMessage_SetExecutionContext); ok { + return x.SetExecutionContext + } + return nil +} + +type isRemoteFileSourceClientMessage_Request interface { + isRemoteFileSourceClientMessage_Request() +} + +type RemoteFileSourceClientMessage_MetaResponse struct { + // Response to a resource request + MetaResponse *RemoteFileSourceMetaResponse `protobuf:"bytes,2,opt,name=meta_response,json=metaResponse,proto3,oneof"` +} + +type RemoteFileSourceClientMessage_SetExecutionContext struct { + // Set the execution context for script execution + SetExecutionContext *SetExecutionContextRequest `protobuf:"bytes,3,opt,name=set_execution_context,json=setExecutionContext,proto3,oneof"` +} + +func (*RemoteFileSourceClientMessage_MetaResponse) isRemoteFileSourceClientMessage_Request() {} + +func (*RemoteFileSourceClientMessage_SetExecutionContext) isRemoteFileSourceClientMessage_Request() {} + +// Request source data/resource from the client +type RemoteFileSourceMetaRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The name/path of the resource being requested (e.g., "com/example/MyClass.java") + ResourceName string `protobuf:"bytes,1,opt,name=resource_name,json=resourceName,proto3" json:"resource_name,omitempty"` +} + +func (x *RemoteFileSourceMetaRequest) Reset() { + *x = RemoteFileSourceMetaRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoteFileSourceMetaRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoteFileSourceMetaRequest) ProtoMessage() {} + +func (x *RemoteFileSourceMetaRequest) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoteFileSourceMetaRequest.ProtoReflect.Descriptor instead. +func (*RemoteFileSourceMetaRequest) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{2} +} + +func (x *RemoteFileSourceMetaRequest) GetResourceName() string { + if x != nil { + return x.ResourceName + } + return "" +} + +// Response to a resource request +type RemoteFileSourceMetaResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The content of the resource, or empty if not found + Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + // Indicates whether the resource was found + Found bool `protobuf:"varint,2,opt,name=found,proto3" json:"found,omitempty"` + // Error message if the resource could not be retrieved + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` +} + +func (x *RemoteFileSourceMetaResponse) Reset() { + *x = RemoteFileSourceMetaResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoteFileSourceMetaResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoteFileSourceMetaResponse) ProtoMessage() {} + +func (x *RemoteFileSourceMetaResponse) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoteFileSourceMetaResponse.ProtoReflect.Descriptor instead. +func (*RemoteFileSourceMetaResponse) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{3} +} + +func (x *RemoteFileSourceMetaResponse) GetContent() []byte { + if x != nil { + return x.Content + } + return nil +} + +func (x *RemoteFileSourceMetaResponse) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *RemoteFileSourceMetaResponse) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +// Request to set the execution context for script execution +type SetExecutionContextRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Resource paths that should be resolved from the remote source + // (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) + ResourcePaths []string `protobuf:"bytes,1,rep,name=resource_paths,json=resourcePaths,proto3" json:"resource_paths,omitempty"` + // Indicates whether remote sources have changed and cache should be cleared + IsDirty bool `protobuf:"varint,2,opt,name=is_dirty,json=isDirty,proto3" json:"is_dirty,omitempty"` +} + +func (x *SetExecutionContextRequest) Reset() { + *x = SetExecutionContextRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SetExecutionContextRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetExecutionContextRequest) ProtoMessage() {} + +func (x *SetExecutionContextRequest) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetExecutionContextRequest.ProtoReflect.Descriptor instead. +func (*SetExecutionContextRequest) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{4} +} + +func (x *SetExecutionContextRequest) GetResourcePaths() []string { + if x != nil { + return x.ResourcePaths + } + return nil +} + +func (x *SetExecutionContextRequest) GetIsDirty() bool { + if x != nil { + return x.IsDirty + } + return false +} + +// Response acknowledging execution context was set +type SetExecutionContextResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Whether the operation was successful + Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` +} + +func (x *SetExecutionContextResponse) Reset() { + *x = SetExecutionContextResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SetExecutionContextResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetExecutionContextResponse) ProtoMessage() {} + +func (x *SetExecutionContextResponse) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetExecutionContextResponse.ProtoReflect.Descriptor instead. +func (*SetExecutionContextResponse) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{5} +} + +func (x *SetExecutionContextResponse) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +// Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream) +type RemoteFileSourcePluginFetchRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ResultId *ticket.Ticket `protobuf:"bytes,1,opt,name=result_id,json=resultId,proto3" json:"result_id,omitempty"` + // The plugin name to create the PluginMarker for. + PluginName string `protobuf:"bytes,2,opt,name=plugin_name,json=pluginName,proto3" json:"plugin_name,omitempty"` +} + +func (x *RemoteFileSourcePluginFetchRequest) Reset() { + *x = RemoteFileSourcePluginFetchRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemoteFileSourcePluginFetchRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemoteFileSourcePluginFetchRequest) ProtoMessage() {} + +func (x *RemoteFileSourcePluginFetchRequest) ProtoReflect() protoreflect.Message { + mi := &file_deephaven_core_proto_remotefilesource_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemoteFileSourcePluginFetchRequest.ProtoReflect.Descriptor instead. +func (*RemoteFileSourcePluginFetchRequest) Descriptor() ([]byte, []int) { + return file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP(), []int{6} +} + +func (x *RemoteFileSourcePluginFetchRequest) GetResultId() *ticket.Ticket { + if x != nil { + return x.ResultId + } + return nil +} + +func (x *RemoteFileSourcePluginFetchRequest) GetPluginName() string { + if x != nil { + return x.PluginName + } + return "" +} + +var File_deephaven_core_proto_remotefilesource_proto protoreflect.FileDescriptor + +var file_deephaven_core_proto_remotefilesource_proto_rawDesc = []byte{ + 0x0a, 0x2b, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, 0x6e, 0x5f, 0x63, 0x6f, 0x72, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x66, 0x69, 0x6c, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x69, + 0x6f, 0x2e, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x62, 0x61, 0x63, 0x6b, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x1a, 0x21, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, 0x6e, 0x5f, 0x63, 0x6f, 0x72, 0x65, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0xb6, 0x02, 0x0a, 0x1d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x69, + 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x49, 0x64, 0x12, 0x63, 0x0a, 0x0c, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x69, 0x6f, 0x2e, + 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x62, 0x61, 0x63, 0x6b, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, + 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x6d, 0x65, + 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x85, 0x01, 0x0a, 0x1e, 0x73, 0x65, + 0x74, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x78, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, + 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x62, 0x61, 0x63, 0x6b, 0x70, 0x6c, 0x61, 0x6e, + 0x65, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x48, 0x00, 0x52, 0x1b, 0x73, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xa6, 0x02, 0x0a, + 0x1d, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x66, 0x0a, + 0x0d, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, + 0x76, 0x65, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x62, 0x61, 0x63, 0x6b, 0x70, 0x6c, + 0x61, 0x6e, 0x65, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x73, 0x0a, 0x15, 0x73, 0x65, 0x74, 0x5f, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, + 0x76, 0x65, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x62, 0x61, 0x63, 0x6b, 0x70, 0x6c, + 0x61, 0x6e, 0x65, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x13, 0x73, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x42, 0x0a, 0x1b, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x64, 0x0a, 0x1c, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4d, 0x65, 0x74, + 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, + 0x5e, 0x0a, 0x1a, 0x53, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, + 0x0e, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, + 0x61, 0x74, 0x68, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x64, 0x69, 0x72, 0x74, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x44, 0x69, 0x72, 0x74, 0x79, 0x22, + 0x37, 0x0a, 0x1b, 0x53, 0x65, 0x74, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x8d, 0x01, 0x0a, 0x22, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x6c, 0x75, + 0x67, 0x69, 0x6e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x46, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, + 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x62, 0x61, 0x63, 0x6b, 0x70, 0x6c, 0x61, 0x6e, + 0x65, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x54, 0x69, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x08, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x6c, + 0x75, 0x67, 0x69, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x4c, 0x48, 0x01, 0x50, 0x01, 0x5a, 0x46, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x65, 0x65, 0x70, 0x68, + 0x61, 0x76, 0x65, 0x6e, 0x2f, 0x64, 0x65, 0x65, 0x70, 0x68, 0x61, 0x76, 0x65, 0x6e, 0x2d, 0x63, + 0x6f, 0x72, 0x65, 0x2f, 0x67, 0x6f, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x66, 0x69, 0x6c, 0x65, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_deephaven_core_proto_remotefilesource_proto_rawDescOnce sync.Once + file_deephaven_core_proto_remotefilesource_proto_rawDescData = file_deephaven_core_proto_remotefilesource_proto_rawDesc +) + +func file_deephaven_core_proto_remotefilesource_proto_rawDescGZIP() []byte { + file_deephaven_core_proto_remotefilesource_proto_rawDescOnce.Do(func() { + file_deephaven_core_proto_remotefilesource_proto_rawDescData = protoimpl.X.CompressGZIP(file_deephaven_core_proto_remotefilesource_proto_rawDescData) + }) + return file_deephaven_core_proto_remotefilesource_proto_rawDescData +} + +var file_deephaven_core_proto_remotefilesource_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_deephaven_core_proto_remotefilesource_proto_goTypes = []interface{}{ + (*RemoteFileSourceServerMessage)(nil), // 0: io.deephaven.proto.backplane.grpc.RemoteFileSourceServerMessage + (*RemoteFileSourceClientMessage)(nil), // 1: io.deephaven.proto.backplane.grpc.RemoteFileSourceClientMessage + (*RemoteFileSourceMetaRequest)(nil), // 2: io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest + (*RemoteFileSourceMetaResponse)(nil), // 3: io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse + (*SetExecutionContextRequest)(nil), // 4: io.deephaven.proto.backplane.grpc.SetExecutionContextRequest + (*SetExecutionContextResponse)(nil), // 5: io.deephaven.proto.backplane.grpc.SetExecutionContextResponse + (*RemoteFileSourcePluginFetchRequest)(nil), // 6: io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest + (*ticket.Ticket)(nil), // 7: io.deephaven.proto.backplane.grpc.Ticket +} +var file_deephaven_core_proto_remotefilesource_proto_depIdxs = []int32{ + 2, // 0: io.deephaven.proto.backplane.grpc.RemoteFileSourceServerMessage.meta_request:type_name -> io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequest + 5, // 1: io.deephaven.proto.backplane.grpc.RemoteFileSourceServerMessage.set_execution_context_response:type_name -> io.deephaven.proto.backplane.grpc.SetExecutionContextResponse + 3, // 2: io.deephaven.proto.backplane.grpc.RemoteFileSourceClientMessage.meta_response:type_name -> io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponse + 4, // 3: io.deephaven.proto.backplane.grpc.RemoteFileSourceClientMessage.set_execution_context:type_name -> io.deephaven.proto.backplane.grpc.SetExecutionContextRequest + 7, // 4: io.deephaven.proto.backplane.grpc.RemoteFileSourcePluginFetchRequest.result_id:type_name -> io.deephaven.proto.backplane.grpc.Ticket + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_deephaven_core_proto_remotefilesource_proto_init() } +func file_deephaven_core_proto_remotefilesource_proto_init() { + if File_deephaven_core_proto_remotefilesource_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_deephaven_core_proto_remotefilesource_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoteFileSourceServerMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoteFileSourceClientMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoteFileSourceMetaRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoteFileSourceMetaResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SetExecutionContextRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SetExecutionContextResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemoteFileSourcePluginFetchRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*RemoteFileSourceServerMessage_MetaRequest)(nil), + (*RemoteFileSourceServerMessage_SetExecutionContextResponse)(nil), + } + file_deephaven_core_proto_remotefilesource_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*RemoteFileSourceClientMessage_MetaResponse)(nil), + (*RemoteFileSourceClientMessage_SetExecutionContext)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_deephaven_core_proto_remotefilesource_proto_rawDesc, + NumEnums: 0, + NumMessages: 7, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_deephaven_core_proto_remotefilesource_proto_goTypes, + DependencyIndexes: file_deephaven_core_proto_remotefilesource_proto_depIdxs, + MessageInfos: file_deephaven_core_proto_remotefilesource_proto_msgTypes, + }.Build() + File_deephaven_core_proto_remotefilesource_proto = out.File + file_deephaven_core_proto_remotefilesource_proto_rawDesc = nil + file_deephaven_core_proto_remotefilesource_proto_goTypes = nil + file_deephaven_core_proto_remotefilesource_proto_depIdxs = nil +} diff --git a/py/client/deephaven_core/proto/remotefilesource_pb2.py b/py/client/deephaven_core/proto/remotefilesource_pb2.py new file mode 100644 index 00000000000..3aa205c62e3 --- /dev/null +++ b/py/client/deephaven_core/proto/remotefilesource_pb2.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: deephaven_core/proto/remotefilesource.proto +# Protobuf Python Version: 6.31.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 31, + 1, + '', + 'deephaven_core/proto/remotefilesource.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from deephaven_core.proto import ticket_pb2 as deephaven__core_dot_proto_dot_ticket__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n+deephaven_core/proto/remotefilesource.proto\x12!io.deephaven.proto.backplane.grpc\x1a!deephaven_core/proto/ticket.proto\"\x80\x02\n\x1dRemoteFileSourceServerMessage\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12V\n\x0cmeta_request\x18\x02 \x01(\x0b\x32>.io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaRequestH\x00\x12h\n\x1eset_execution_context_response\x18\x03 \x01(\x0b\x32>.io.deephaven.proto.backplane.grpc.SetExecutionContextResponseH\x00\x42\t\n\x07request\"\xf8\x01\n\x1dRemoteFileSourceClientMessage\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12X\n\rmeta_response\x18\x02 \x01(\x0b\x32?.io.deephaven.proto.backplane.grpc.RemoteFileSourceMetaResponseH\x00\x12^\n\x15set_execution_context\x18\x03 \x01(\x0b\x32=.io.deephaven.proto.backplane.grpc.SetExecutionContextRequestH\x00\x42\t\n\x07request\"4\n\x1bRemoteFileSourceMetaRequest\x12\x15\n\rresource_name\x18\x01 \x01(\t\"M\n\x1cRemoteFileSourceMetaResponse\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\r\n\x05\x66ound\x18\x02 \x01(\x08\x12\r\n\x05\x65rror\x18\x03 \x01(\t\"F\n\x1aSetExecutionContextRequest\x12\x16\n\x0eresource_paths\x18\x01 \x03(\t\x12\x10\n\x08is_dirty\x18\x02 \x01(\x08\".\n\x1bSetExecutionContextResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\"w\n\"RemoteFileSourcePluginFetchRequest\x12<\n\tresult_id\x18\x01 \x01(\x0b\x32).io.deephaven.proto.backplane.grpc.Ticket\x12\x13\n\x0bplugin_name\x18\x02 \x01(\tBLH\x01P\x01ZFgithub.com/deephaven/deephaven-core/go/internal/proto/remotefilesourceb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'deephaven_core.proto.remotefilesource_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'H\001P\001ZFgithub.com/deephaven/deephaven-core/go/internal/proto/remotefilesource' + _globals['_REMOTEFILESOURCESERVERMESSAGE']._serialized_start=118 + _globals['_REMOTEFILESOURCESERVERMESSAGE']._serialized_end=374 + _globals['_REMOTEFILESOURCECLIENTMESSAGE']._serialized_start=377 + _globals['_REMOTEFILESOURCECLIENTMESSAGE']._serialized_end=625 + _globals['_REMOTEFILESOURCEMETAREQUEST']._serialized_start=627 + _globals['_REMOTEFILESOURCEMETAREQUEST']._serialized_end=679 + _globals['_REMOTEFILESOURCEMETARESPONSE']._serialized_start=681 + _globals['_REMOTEFILESOURCEMETARESPONSE']._serialized_end=758 + _globals['_SETEXECUTIONCONTEXTREQUEST']._serialized_start=760 + _globals['_SETEXECUTIONCONTEXTREQUEST']._serialized_end=830 + _globals['_SETEXECUTIONCONTEXTRESPONSE']._serialized_start=832 + _globals['_SETEXECUTIONCONTEXTRESPONSE']._serialized_end=878 + _globals['_REMOTEFILESOURCEPLUGINFETCHREQUEST']._serialized_start=880 + _globals['_REMOTEFILESOURCEPLUGINFETCHREQUEST']._serialized_end=999 +# @@protoc_insertion_point(module_scope) diff --git a/py/client/deephaven_core/proto/remotefilesource_pb2.pyi b/py/client/deephaven_core/proto/remotefilesource_pb2.pyi new file mode 100644 index 00000000000..08689044308 --- /dev/null +++ b/py/client/deephaven_core/proto/remotefilesource_pb2.pyi @@ -0,0 +1,211 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file + +Copyright (c) 2016-2025 Deephaven Data Labs and Patent Pending +""" + +import builtins +import collections.abc +import deephaven_core.proto.ticket_pb2 +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.message +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +@typing.final +class RemoteFileSourceServerMessage(google.protobuf.message.Message): + """Server → Client: Requests sent from server to client via MessageStream""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REQUEST_ID_FIELD_NUMBER: builtins.int + META_REQUEST_FIELD_NUMBER: builtins.int + SET_EXECUTION_CONTEXT_RESPONSE_FIELD_NUMBER: builtins.int + request_id: builtins.str + """Unique identifier for this request, used to correlate the response""" + @property + def meta_request(self) -> Global___RemoteFileSourceMetaRequest: + """Request source data/resource from the client""" + + @property + def set_execution_context_response(self) -> Global___SetExecutionContextResponse: + """Acknowledgment that execution context was set""" + + def __init__( + self, + *, + request_id: builtins.str = ..., + meta_request: Global___RemoteFileSourceMetaRequest | None = ..., + set_execution_context_response: Global___SetExecutionContextResponse | None = ..., + ) -> None: ... + _HasFieldArgType: typing_extensions.TypeAlias = typing.Literal["meta_request", b"meta_request", "request", b"request", "set_execution_context_response", b"set_execution_context_response"] + def HasField(self, field_name: _HasFieldArgType) -> builtins.bool: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["meta_request", b"meta_request", "request", b"request", "request_id", b"request_id", "set_execution_context_response", b"set_execution_context_response"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + _WhichOneofReturnType_request: typing_extensions.TypeAlias = typing.Literal["meta_request", "set_execution_context_response"] + _WhichOneofArgType_request: typing_extensions.TypeAlias = typing.Literal["request", b"request"] + def WhichOneof(self, oneof_group: _WhichOneofArgType_request) -> _WhichOneofReturnType_request | None: ... + +Global___RemoteFileSourceServerMessage: typing_extensions.TypeAlias = RemoteFileSourceServerMessage + +@typing.final +class RemoteFileSourceClientMessage(google.protobuf.message.Message): + """Client → Server: Requests/responses sent from client to server via MessageStream""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + REQUEST_ID_FIELD_NUMBER: builtins.int + META_RESPONSE_FIELD_NUMBER: builtins.int + SET_EXECUTION_CONTEXT_FIELD_NUMBER: builtins.int + request_id: builtins.str + """The request_id from the ServerRequest this is responding to (if applicable)""" + @property + def meta_response(self) -> Global___RemoteFileSourceMetaResponse: + """Response to a resource request""" + + @property + def set_execution_context(self) -> Global___SetExecutionContextRequest: + """Set the execution context for script execution""" + + def __init__( + self, + *, + request_id: builtins.str = ..., + meta_response: Global___RemoteFileSourceMetaResponse | None = ..., + set_execution_context: Global___SetExecutionContextRequest | None = ..., + ) -> None: ... + _HasFieldArgType: typing_extensions.TypeAlias = typing.Literal["meta_response", b"meta_response", "request", b"request", "set_execution_context", b"set_execution_context"] + def HasField(self, field_name: _HasFieldArgType) -> builtins.bool: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["meta_response", b"meta_response", "request", b"request", "request_id", b"request_id", "set_execution_context", b"set_execution_context"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + _WhichOneofReturnType_request: typing_extensions.TypeAlias = typing.Literal["meta_response", "set_execution_context"] + _WhichOneofArgType_request: typing_extensions.TypeAlias = typing.Literal["request", b"request"] + def WhichOneof(self, oneof_group: _WhichOneofArgType_request) -> _WhichOneofReturnType_request | None: ... + +Global___RemoteFileSourceClientMessage: typing_extensions.TypeAlias = RemoteFileSourceClientMessage + +@typing.final +class RemoteFileSourceMetaRequest(google.protobuf.message.Message): + """Request source data/resource from the client""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RESOURCE_NAME_FIELD_NUMBER: builtins.int + resource_name: builtins.str + """The name/path of the resource being requested (e.g., "com/example/MyClass.java")""" + def __init__( + self, + *, + resource_name: builtins.str = ..., + ) -> None: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["resource_name", b"resource_name"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + +Global___RemoteFileSourceMetaRequest: typing_extensions.TypeAlias = RemoteFileSourceMetaRequest + +@typing.final +class RemoteFileSourceMetaResponse(google.protobuf.message.Message): + """Response to a resource request""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + CONTENT_FIELD_NUMBER: builtins.int + FOUND_FIELD_NUMBER: builtins.int + ERROR_FIELD_NUMBER: builtins.int + content: builtins.bytes + """The content of the resource, or empty if not found""" + found: builtins.bool + """Indicates whether the resource was found""" + error: builtins.str + """Error message if the resource could not be retrieved""" + def __init__( + self, + *, + content: builtins.bytes = ..., + found: builtins.bool = ..., + error: builtins.str = ..., + ) -> None: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["content", b"content", "error", b"error", "found", b"found"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + +Global___RemoteFileSourceMetaResponse: typing_extensions.TypeAlias = RemoteFileSourceMetaResponse + +@typing.final +class SetExecutionContextRequest(google.protobuf.message.Message): + """Request to set the execution context for script execution""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RESOURCE_PATHS_FIELD_NUMBER: builtins.int + IS_DIRTY_FIELD_NUMBER: builtins.int + is_dirty: builtins.bool + """Indicates whether remote sources have changed and cache should be cleared""" + @property + def resource_paths(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: + """Resource paths that should be resolved from the remote source + (e.g., ["com/example/Test.groovy", "org/mycompany/Utils.groovy"]) + """ + + def __init__( + self, + *, + resource_paths: collections.abc.Iterable[builtins.str] | None = ..., + is_dirty: builtins.bool = ..., + ) -> None: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["is_dirty", b"is_dirty", "resource_paths", b"resource_paths"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + +Global___SetExecutionContextRequest: typing_extensions.TypeAlias = SetExecutionContextRequest + +@typing.final +class SetExecutionContextResponse(google.protobuf.message.Message): + """Response acknowledging execution context was set""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SUCCESS_FIELD_NUMBER: builtins.int + success: builtins.bool + """Whether the operation was successful""" + def __init__( + self, + *, + success: builtins.bool = ..., + ) -> None: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["success", b"success"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + +Global___SetExecutionContextResponse: typing_extensions.TypeAlias = SetExecutionContextResponse + +@typing.final +class RemoteFileSourcePluginFetchRequest(google.protobuf.message.Message): + """Fetch the remote file source plugin into the specified ticket (Flight command, not MessageStream)""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RESULT_ID_FIELD_NUMBER: builtins.int + PLUGIN_NAME_FIELD_NUMBER: builtins.int + plugin_name: builtins.str + """The plugin name to create the PluginMarker for.""" + @property + def result_id(self) -> deephaven_core.proto.ticket_pb2.Ticket: ... + def __init__( + self, + *, + result_id: deephaven_core.proto.ticket_pb2.Ticket | None = ..., + plugin_name: builtins.str = ..., + ) -> None: ... + _HasFieldArgType: typing_extensions.TypeAlias = typing.Literal["result_id", b"result_id"] + def HasField(self, field_name: _HasFieldArgType) -> builtins.bool: ... + _ClearFieldArgType: typing_extensions.TypeAlias = typing.Literal["plugin_name", b"plugin_name", "result_id", b"result_id"] + def ClearField(self, field_name: _ClearFieldArgType) -> None: ... + +Global___RemoteFileSourcePluginFetchRequest: typing_extensions.TypeAlias = RemoteFileSourcePluginFetchRequest diff --git a/py/client/deephaven_core/proto/remotefilesource_pb2_grpc.py b/py/client/deephaven_core/proto/remotefilesource_pb2_grpc.py new file mode 100644 index 00000000000..a66295078b8 --- /dev/null +++ b/py/client/deephaven_core/proto/remotefilesource_pb2_grpc.py @@ -0,0 +1,24 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + + +GRPC_GENERATED_VERSION = '1.76.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + ' but the generated code in deephaven_core/proto/remotefilesource_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) From 96645043d5b58bf8281331e723612355b029594a Mon Sep 17 00:00:00 2001 From: Brian Ingles Date: Fri, 13 Mar 2026 10:28:42 -0500 Subject: [PATCH 78/78] Added isDirty flag (#DH-20578) --- .../remotefilesource/JsRemoteFileSourceService.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java index f7434e0f5bc..ec51fb748ed 100644 --- a/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java +++ b/web/client-api/src/main/java/io/deephaven/web/client/api/remotefilesource/JsRemoteFileSourceService.java @@ -233,12 +233,13 @@ private void handleSetExecutionContextResponse(RemoteFileSourceServerMessage mes /** * Sets the execution context on the server to identify this message stream as active for script execution. * + * @param isDirty whether the execution context is dirty (has pending changes) * @param resourcePaths array of resource paths to resolve from remote source (e.g., ["com/example/Test.groovy", * "org/mycompany/Utils.groovy"]), or null/empty for no specific resources * @return a promise that resolves to true if the server successfully set the execution context, false otherwise */ @JsMethod - public Promise setExecutionContext(@JsOptional String[] resourcePaths) { + public Promise setExecutionContext(boolean isDirty, @JsOptional String[] resourcePaths) { // Generate a unique request ID String requestId = "setExecutionContext-" + (requestIdCounter++); @@ -247,7 +248,7 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) pendingSetExecutionContextRequests.put(requestId, promise); // Send the request - RemoteFileSourceClientMessage clientRequest = getSetExecutionContextRequest(resourcePaths, requestId); + RemoteFileSourceClientMessage clientRequest = getSetExecutionContextRequest(isDirty, resourcePaths, requestId); sendClientRequest(clientRequest); // Return a promise with built-in timeout @@ -257,13 +258,15 @@ public Promise setExecutionContext(@JsOptional String[] resourcePaths) /** * Helper method to build a RemoteFileSourceClientMessage for setting execution context. * + * @param isDirty whether the execution context is dirty (has pending changes) * @param resourcePaths array of resource paths to resolve * @param requestId unique request ID * @return the constructed RemoteFileSourceClientMessage */ - private static @NotNull RemoteFileSourceClientMessage getSetExecutionContextRequest(String[] resourcePaths, - String requestId) { + private static @NotNull RemoteFileSourceClientMessage getSetExecutionContextRequest(boolean isDirty, + String[] resourcePaths, String requestId) { SetExecutionContextRequest setContextRequest = new SetExecutionContextRequest(); + setContextRequest.setIsDirty(isDirty); if (resourcePaths != null) { setContextRequest.setResourcePathsList(resourcePaths);