diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..462c85a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM selenium/hub:3.14 AS base + +FROM gradle:5.4.1-jdk8 AS gradle + +RUN apt update && apt install dos2unix +COPY . . +RUN dos2unix gradlew +RUN ./gradlew clean jar + +FROM base AS selenium + +RUN sudo sed -i 's/\-jar \/opt\/selenium\/selenium-server-standalone\.jar'\ +'/ -cp \/opt\/selenium\/selenium-server-standalone\.jar:\/opt\/selenium\/selenium-api.jar org.openqa.grid.selenium.GridLauncherV3 -servlets com.xing.qa.selenium.grid.hub.Console/'\ + /opt/bin/start-selenium-hub.sh +COPY --from=gradle /home/gradle/build/libs/*.jar /opt/selenium/selenium-api.jar \ No newline at end of file diff --git a/README.md b/README.md index 78fd0bd..5c79a05 100644 --- a/README.md +++ b/README.md @@ -343,3 +343,12 @@ you can map your matcher to a capability (name) in the Matcher configuration. * The configuration is somewhat elaborate to avoid messing with the core selenium configuration. It might be an idea to add a configuration file on its own or to extend the selenium node configuration as the configuration by environment might hit a limit at some point (esp. with the ConfigurableCapabilityMatcher). + +## Docker / Kubernetes + +Build and run in Docker. (Removes the need for JDK/JRE/Gradle etc.) +```Bash +docker build . -t selenium-api +docker run -it -p 4444:4444 selenium-api +``` +A precompiled image is available on [Dock Hub](https://hub.docker.com/r/joes88/selenium-api) \ No newline at end of file diff --git a/build.gradle b/build.gradle index c1a2c83..265fd16 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,7 @@ dependencies { provided 'org.seleniumhq.selenium:selenium-server:3.14.0' compile 'org.json:json:20080701' testCompile 'org.testng:testng:6.8.8' + compile 'org.influxdb:influxdb-java:2.11' } jar { diff --git a/src/main/java/com/xing/qa/selenium/grid/hub/Console.java b/src/main/java/com/xing/qa/selenium/grid/hub/Console.java index 7b9568a..2aaca4b 100644 --- a/src/main/java/com/xing/qa/selenium/grid/hub/Console.java +++ b/src/main/java/com/xing/qa/selenium/grid/hub/Console.java @@ -8,6 +8,7 @@ import org.openqa.grid.web.servlet.RegistryBasedServlet; import org.openqa.selenium.BuildInfo; import org.openqa.selenium.Capabilities; +import org.openqa.selenium.Proxy; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -109,12 +110,14 @@ protected JSONObject status() Hub h = getRegistry().getHub(); List nodes = new ArrayList<>(); - for (RemoteProxy proxy : getRegistry().getAllProxies()) { try { JSONRenderer beta = new WebProxyJsonRenderer(proxy); nodes.add(beta.render()); - } catch (Exception e) {} + } catch (Exception e) { + System.out.println(e.getMessage()); + e.printStackTrace(); + } } status.put("version", coreVersion); diff --git a/src/main/java/com/xing/qa/selenium/grid/hub/WebProxyJsonRenderer.java b/src/main/java/com/xing/qa/selenium/grid/hub/WebProxyJsonRenderer.java index 3b26eab..4ab157e 100644 --- a/src/main/java/com/xing/qa/selenium/grid/hub/WebProxyJsonRenderer.java +++ b/src/main/java/com/xing/qa/selenium/grid/hub/WebProxyJsonRenderer.java @@ -46,7 +46,7 @@ public JSONObject render() throws JSONException { } json.put("os", value.get("os")); - json.put("java", java.get("version")); + json.put("java", java == null ? "none" : java.get("version")); json.put("configuration", proxy.getConfig()); SlotsLines rcLines = new SlotsLines(); @@ -86,7 +86,9 @@ private JSONObject render(TestSlot slot) throws JSONException { json.put("capabilities", slot.getCapabilities()); TestSession session = slot.getSession(); - if (session != null) { + if (session != null + && session.getExternalKey() != null // Session creation failed https://github.com/SeleniumHQ/selenium/blob/49dc495acaf53bb5f612e2e1552dc5dc40d1c62f/java/server/src/org/openqa/grid/internal/ActiveTestSessions.java#L77 + ) { json.put("session", render(session)); json.put("busy", true); } else { diff --git a/src/main/java/com/xing/qa/selenium/grid/node/BaseSeleniumReporter.java b/src/main/java/com/xing/qa/selenium/grid/node/BaseSeleniumReporter.java new file mode 100644 index 0000000..efeec94 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/BaseSeleniumReporter.java @@ -0,0 +1,46 @@ +package com.xing.qa.selenium.grid.node; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.openqa.grid.internal.ExternalSessionKey; +import org.openqa.grid.internal.TestSession; + +import javax.servlet.http.HttpServletResponse; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * BaseSeleniumReporter + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +public abstract class BaseSeleniumReporter implements Runnable { + + protected final String remoteHostName; + protected final Logger log = Logger.getLogger(getClass().getName()); + private final String database; + private final InfluxDB influxdb; + + public BaseSeleniumReporter(String remoteHostName, InfluxDB influxdb, String database) { + this.remoteHostName = remoteHostName; + this.influxdb = influxdb; + this.database = database; + } + + @Override + public final void run() { + try { + report(); + } catch (Exception e) { + log.warning(e.getMessage()); + } + } + + protected abstract void report(); + + protected void write(Point... points) { + for (Point p : points) { + influxdb.write(p); + } + } +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/CommandReporter.java b/src/main/java/com/xing/qa/selenium/grid/node/CommandReporter.java new file mode 100644 index 0000000..0a23993 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/CommandReporter.java @@ -0,0 +1,52 @@ +package com.xing.qa.selenium.grid.node; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.openqa.grid.internal.ExternalSessionKey; +import org.openqa.grid.internal.TestSession; + +import javax.servlet.http.HttpServletResponse; +import java.util.concurrent.TimeUnit; + +/** + * CommandReporter + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +class CommandReporter extends BaseSeleniumReporter { + + protected final TestSession session; + protected final ContentSnoopingRequest request; + protected final HttpServletResponse response; + protected final ReportType type; + + public CommandReporter(String remoteHostName, InfluxDB influxdb, String database, TestSession session, ContentSnoopingRequest request, HttpServletResponse response, ReportType type) { + super(remoteHostName, influxdb, database); + this.type = type; + this.request = request; + this.session = session; + this.response = response; + } + + protected void report() { + ExternalSessionKey esk = session.getExternalKey(); + String sessionKey = null; + if (esk != null) { + sessionKey = esk.getKey(); + } + + Point s = Point.measurement(String.format("session.cmd.%s.measure", type)) + .addField("time",System.currentTimeMillis()) + .addField("host",remoteHostName) + .addField("ext_key",sessionKey) + .addField("int_key",session.getInternalKey()) + .addField("forwarding",session.isForwardingRequest()) + .addField("orphaned",session.isOrphaned()) + .addField("inactivity",session.getInactivityTime()) + .addField("cmd_method",request.getMethod()) + .addField("cmd_action",request.getPathInfo()) + .addField("cmd",request.getContent()) + .build(); + write(s); + } +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/ContentSnoopingRequest.java b/src/main/java/com/xing/qa/selenium/grid/node/ContentSnoopingRequest.java new file mode 100644 index 0000000..d1b5197 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/ContentSnoopingRequest.java @@ -0,0 +1,86 @@ +package com.xing.qa.selenium.grid.node; + +import javax.servlet.ReadListener; +import javax.servlet.ServletInputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletRequestWrapper; +import java.io.IOException; +import java.io.InputStream; + +/** +* ContentSnoopingRequest wraps the original request to allow us to peek into the payload of the request. +* +* @author Jens Hausherr (jens.hausherr@xing.com) +*/ +class ContentSnoopingRequest extends HttpServletRequestWrapper { + + private String content; + private String encoding; + + public String getContent() { + return content; + } + + /** + * Constructs a request object wrapping the given request. + * + * @param request + * @throws IllegalArgumentException if the request is null + */ + public ContentSnoopingRequest(HttpServletRequest request) { + super(request); + + encoding = request.getCharacterEncoding(); + if (encoding == null) + encoding = "ISO-8859-1"; + + try { + StringBuilder sb = new StringBuilder(); + InputStream is = request.getInputStream(); + byte[] buffer = new byte[1024]; + int read = 0; + + while ((read = is.read(buffer, 0, 1024)) != -1) { + sb.append(new String(buffer, 0, read, encoding)); + } + + this.content = sb.toString(); + + } catch (IOException e) { + e.printStackTrace(); + this.content = ""; + } + } + + @Override + public ServletInputStream getInputStream() throws IOException { + return new ServletInputStream() { + + int idx = 0; + byte[] contents = content.getBytes(encoding); + + @Override + public boolean isFinished() { + return idx < contents.length; + } + + @Override + public boolean isReady() { + return !isFinished(); + } + + @Override + public int read() throws IOException { + if (idx < contents.length) { + return contents[idx++]; + } else return -1; + } + + @Override + public void setReadListener(ReadListener listener) { + throw new java.lang.IllegalStateException(); + } + }; + } + +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/ErrorReporter.java b/src/main/java/com/xing/qa/selenium/grid/node/ErrorReporter.java new file mode 100644 index 0000000..fd1a9cd --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/ErrorReporter.java @@ -0,0 +1,33 @@ +package com.xing.qa.selenium.grid.node; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.openqa.grid.common.exception.RemoteException; +import org.openqa.grid.internal.TestSession; + +import javax.servlet.http.HttpServletResponse; +import java.util.concurrent.TimeUnit; + +/** + * ErrorReporter + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +class ErrorReporter extends BaseSeleniumReporter { + + private final RemoteException exception; + + public ErrorReporter(String remoteHostName, InfluxDB influxdb, String database, RemoteException ex) { + super(remoteHostName, influxdb, database); + this.exception = ex; + } + + @Override + protected void report() { + final Point exRep = Point.measurement("node.errors").addField("time", System.currentTimeMillis()) + .addField("host", remoteHostName).addField("error", exception.getClass().getName()) + .addField("message", exception.getMessage()).build(); + write(exRep); + } + +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/MonitoringWebProxy.java b/src/main/java/com/xing/qa/selenium/grid/node/MonitoringWebProxy.java new file mode 100644 index 0000000..3794476 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/MonitoringWebProxy.java @@ -0,0 +1,132 @@ +package com.xing.qa.selenium.grid.node; + +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.openqa.grid.common.RegistrationRequest; +import org.openqa.grid.common.exception.RemoteException; +import org.openqa.grid.internal.GridRegistry; +import org.openqa.grid.internal.TestSession; +import org.openqa.grid.selenium.proxy.DefaultRemoteProxy; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.net.InetAddress; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +import static java.lang.String.format; + +/** + * MonitoringWebProxy + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +public class MonitoringWebProxy extends DefaultRemoteProxy { + + private static final Logger LOG = Logger.getLogger(MonitoringWebProxy.class.getName()); + + private static final String DATABASE = envOr("IFXDB_DB", "selenium-grid"); + + private static final ScheduledExecutorService EXECUTOR = Executors.newScheduledThreadPool(64); + + private static final String URL = format("http://%s:%s", envOr("IFXDB_HOST", "localhost"), + envOr("IFXDB_PORT", "8086")); + private static final InfluxDB INFLUX_DB = InfluxDBFactory.connect(URL, envOr("IFXDB_USER", "root"), + envOr("IFXDB_PASSWD", "root")); + + static { + LOG.info(String.format("Reporting to: %s/db/%s", URL, DATABASE)); + INFLUX_DB.setLogLevel(InfluxDB.LogLevel.NONE); + INFLUX_DB.setDatabase(DATABASE); + } + + private static String envOr(String envVar, String defaultVal) { + String val = System.getenv(envVar); + if (val == null) + return defaultVal; + return val; + } + + private final Logger log = Logger.getLogger(getClass().getName()); + + private ScheduledFuture nodeReporter; + + private String remoteHostName; + + public MonitoringWebProxy(RegistrationRequest request, GridRegistry registry) { + super(request, registry); + final String addr = request.getConfiguration().host; + remoteHostName = addr.toLowerCase(); + + try { + remoteHostName = InetAddress.getByName(addr).getCanonicalHostName().toLowerCase(); + } catch (Exception e) { + LOG.info(format("Unable to lookup name for remote address '%s", addr)); + } + + LOG.info(String.format("Initializing monitoring WebProxy for %s: %s.", remoteHostName, request.toJson())); + + nodeReporter = EXECUTOR.scheduleAtFixedRate(new NodeReporter(remoteHostName, INFLUX_DB, DATABASE, this), 0, 5, + TimeUnit.SECONDS); + } + + @Override + public void beforeSession(TestSession session) { + EXECUTOR.execute(new SessionReporter(remoteHostName, INFLUX_DB, DATABASE, session, ReportType.start)); + super.beforeSession(session); + } + + @Override + public void afterSession(TestSession session) { + super.afterSession(session); + EXECUTOR.execute(new SessionReporter(remoteHostName, INFLUX_DB, DATABASE, session, ReportType.finish)); + } + + @Override + public void onEvent(List events, RemoteException lastInserted) { + super.onEvent(events, lastInserted); + EXECUTOR.execute(new ErrorReporter(remoteHostName, INFLUX_DB, DATABASE, lastInserted)); + } + + @Override + public void startPolling() { + super.startPolling(); + } + + @Override + public void stopPolling() { + super.stopPolling(); + } + + @Override + public void afterCommand(TestSession session, HttpServletRequest request, HttpServletResponse response) { + ContentSnoopingRequest req = new ContentSnoopingRequest(request); + EXECUTOR.execute( + new CommandReporter(remoteHostName, INFLUX_DB, DATABASE, session, req, response, ReportType.result)); + super.afterCommand(session, req, response); + } + + @Override + public void beforeCommand(TestSession session, HttpServletRequest request, HttpServletResponse response) { + ContentSnoopingRequest req = new ContentSnoopingRequest(request); + EXECUTOR.execute( + new CommandReporter(remoteHostName, INFLUX_DB, DATABASE, session, req, response, ReportType.command)); + super.beforeCommand(session, req, response); + } + + @Override + public void beforeRelease(TestSession session) { + super.beforeRelease(session); + EXECUTOR.execute(new SessionReporter(remoteHostName, INFLUX_DB, DATABASE, session, ReportType.timeout)); + } + + @Override + public void teardown() { + nodeReporter.cancel(false); + } + +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/NodeReporter.java b/src/main/java/com/xing/qa/selenium/grid/node/NodeReporter.java new file mode 100644 index 0000000..4f2f6a1 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/NodeReporter.java @@ -0,0 +1,30 @@ +package com.xing.qa.selenium.grid.node; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; + +/** + * NodeReporter + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +class NodeReporter extends BaseSeleniumReporter { + + private final MonitoringWebProxy proxy; + + public NodeReporter(String remoteHostName, InfluxDB influxdb, String database, + MonitoringWebProxy monitoringWebProxy) { + super(remoteHostName, influxdb, database); + this.proxy = monitoringWebProxy; + } + + @Override + protected void report() { + log.finer(String.format("Reporting: node.%s.measure", SerieNames.utilization)); + final Point load = Point.measurement(String.format("node.%s.measure", SerieNames.utilization)) + .addField("time", System.currentTimeMillis()).addField("host", remoteHostName) + .addField("used", proxy.getTotalUsed()).addField("total", proxy.getMaxNumberOfConcurrentTestSessions()) + .addField("normalized", proxy.getResourceUsageInPercent()).build(); + write(load); + } +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/ReportType.java b/src/main/java/com/xing/qa/selenium/grid/node/ReportType.java new file mode 100644 index 0000000..9b4c924 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/ReportType.java @@ -0,0 +1,14 @@ +package com.xing.qa.selenium.grid.node; + +/** + * ReportType + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +public enum ReportType { + start, + finish, + timeout, + command, + result +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/SerieNames.java b/src/main/java/com/xing/qa/selenium/grid/node/SerieNames.java new file mode 100644 index 0000000..b023d38 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/SerieNames.java @@ -0,0 +1,13 @@ +package com.xing.qa.selenium.grid.node; + +/** + * SerieNames + * + * @author Jens Hausherr (jens.hausherr@xing.com) + */ +public enum SerieNames { + session, + command, + utilization, + node_errors +} diff --git a/src/main/java/com/xing/qa/selenium/grid/node/SessionReporter.java b/src/main/java/com/xing/qa/selenium/grid/node/SessionReporter.java new file mode 100644 index 0000000..e127cc5 --- /dev/null +++ b/src/main/java/com/xing/qa/selenium/grid/node/SessionReporter.java @@ -0,0 +1,96 @@ +package com.xing.qa.selenium.grid.node; + +import org.influxdb.InfluxDB; +import org.influxdb.dto.Point; +import org.openqa.grid.internal.ExternalSessionKey; +import org.openqa.grid.internal.TestSession; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static java.lang.String.format; + +/** +* Session Reporter +* +* @author Jens Hausherr (jens.hausherr@xing.com) +*/ +class SessionReporter extends BaseSeleniumReporter { + private final TestSession session; + private final ReportType type; + private final InfluxDB influxdb; + private final String database; + + public SessionReporter(String remoteHostName, InfluxDB influxdb, String database, TestSession session, ReportType type) { + super(remoteHostName, influxdb, database); + this.session = session; + this.type = type; + this.influxdb = influxdb; + this.database = database; + } + + @Override + protected void report() { + ExternalSessionKey esk = session.getExternalKey(); + String sessionKey = null; + + if (esk != null) { + sessionKey = esk.getKey(); + } + + final Boolean forwardingRequest = session.isForwardingRequest(); + final Boolean orphaned = session.isOrphaned(); + final Long inactivityTime =session.getInactivityTime(); + final long time = System.currentTimeMillis(); + + + final Point.Builder sessionEvent = Point.measurement("session.event.measure") + .time(System.currentTimeMillis(), TimeUnit.MILLISECONDS); + + sessionEvent.addField("time", time) + .addField("host", remoteHostName) + .addField("type", type.toString()) + .addField("ext_key", sessionKey) + .addField("int_key", session.getInternalKey()) + .addField("forwarding", forwardingRequest) + .addField("orphaned", orphaned) + .addField("inactivity", inactivityTime); + if (ReportType.timeout == type) { + sessionEvent.addField("browser_starting", String.valueOf(session.getInternalKey() == null)); + } + + final Point.Builder sessionCapReq = Point.measurement(format("session.cap.requested.%s.measure", type)); + + for (Map.Entry rcap : session.getRequestedCapabilities().entrySet()) { + sessionCapReq.addField("time", time) + .addField("host", remoteHostName) + .addField("type", type.toString()) + .addField("ext_key", sessionKey) + .addField("int_key", session.getInternalKey()) + .addField("forwarding", forwardingRequest) + .addField("orphaned", orphaned) + .addField("inactivity", inactivityTime) + .addField("capability", rcap.getKey()) + .addField("val", rcap.getValue().toString()); //TODO: Check this, used to take an object, now it has to be a string + + } + + final Point.Builder sessionCapProv = Point.measurement(format("session.cap.provided.%s.measure", type)); + + for (Map.Entry scap : session.getSlot().getCapabilities().entrySet()) { + sessionCapReq.addField("time", time) + .addField("host", remoteHostName) + .addField("type", type.toString()) + .addField("ext_key", sessionKey) + .addField("int_key", session.getInternalKey()) + .addField("forwarding", forwardingRequest) + .addField("orphaned", orphaned) + .addField("inactivity", inactivityTime) + .addField("capability", scap.getKey()) + .addField("val", scap.getValue().toString()); //TODO: Check this, used to take an object, now it has to be a string + } + + write(sessionEvent.build(), sessionCapReq.build(), sessionCapProv.build()); + } + +}