From 028a27407b8c89659dcf86d8f5d120ca27a594cb Mon Sep 17 00:00:00 2001 From: Unknow Date: Tue, 24 Mar 2026 19:50:31 +0100 Subject: [PATCH 01/31] test removing copy of read buffer --- .../server/nio/NIOConnectionHandler.java | 2 +- .../java/unknow/server/nio/NIOWorker.java | 8 +-- .../unknow/server/servlet/HttpConnection.java | 5 +- .../servlet/http11/Http11Processor.java | 10 +-- .../server/servlet/http11/RequestDecoder.java | 63 ++++++++++--------- .../server/servlet/http2/Http2Processor.java | 2 +- 6 files changed, 46 insertions(+), 44 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java index 679526c5..447c18b8 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java @@ -40,7 +40,7 @@ default void onHandshakeDone(SSLEngine sslEngine, long now) throws IOException { /** * called after some data has been read * - * @param b the read buffers + * @param b the read buffers (should read all or copy content) * @param now nanoTime * @throws IOException on io exception */ diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java index f75e0d17..04157814 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java @@ -203,14 +203,10 @@ private void doRead(NIOConnection co, long now) throws IOException { toTail(co, now); if (l == 0) return; - buf.flip(); - ByteBuffer data = ByteBuffer.allocate(buf.remaining()); - data.put(buf); - data.flip(); - co.onRead(data, now); + co.onRead(buf.flip(), now); if (l < BUF_LEN) return; - buf.compact(); + buf.clear(); } } diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java index cfbd481d..1d9f1287 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; @@ -81,8 +80,8 @@ public void onHandshakeDone(SSLEngine sslEngine, long now) throws IOException { @Override public final void onRead(ByteBuffer b, long now) throws IOException { if (p == null) { - if (b.remaining() > Http2Processor.PRI.length - && Arrays.equals(b.array(), b.position(), b.position() + Http2Processor.PRI.length, Http2Processor.PRI, 0, Http2Processor.PRI.length)) + int mismatch = Http2Processor.PRI.mismatch(b); + if (mismatch == 24 || mismatch == -1) p = new Http2Processor(this); else p = new Http11Processor(this); diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java index 4d5f0720..02da76ed 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java @@ -59,14 +59,16 @@ private final void process(ByteBuffer b) throws IOException { decode(b); break; case CONTENT: - ByteBuffer slice = b.slice(); if (b.remaining() < contentLength) { contentLength -= b.remaining(); - dec.addContent(slice); - b.position(b.limit()); + ByteBuffer data = ByteBuffer.allocate(b.remaining()); + data.put(b); + dec.addContent(data); } else { state = START; - dec.addContent(slice.limit((int) contentLength)); + ByteBuffer data = ByteBuffer.allocate((int) contentLength); + data.put(b.slice().limit((int) contentLength)); + dec.addContent(data); dec.closeContent(); b.position(b.position() + (int) contentLength); contentLength = 0; diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java index 452e8636..1a94c581 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java @@ -16,14 +16,18 @@ private enum State { private final Http11Processor co; private final Utf8Decoder decoder; + private final byte[] buf; private ServletRequestImpl req; private State state; private int f; + private int o; + private int l; public RequestDecoder(Http11Processor co) { this.co = co; decoder = new Utf8Decoder(new StringBuilder()); + buf = new byte[4096]; reset(); } @@ -41,24 +45,32 @@ public boolean closed() { public ServletRequestImpl append(ByteBuffer b) { while (b.hasRemaining()) { - tryDecode(b); - if (state == State.DONE) - return req; + int s = b.position(); + l = Math.min(b.remaining(), buf.length); + b.get(buf, 0, l); + o = 0; + while (o < l) { + tryDecode(); + if (state == State.DONE) { + b.position(s + o); + return req; + } + } } return null; } - private void tryDecode(ByteBuffer b) { + private void tryDecode() { String str; switch (state) { case METHOD: - if ((str = readUntil(b, SPACE)) != null) { + if ((str = readUntil(SPACE)) != null) { state = State.URI; req.setMethod(str); } return; case URI: - if ((str = readUntil(b, SPACE)) != null) { + if ((str = readUntil(SPACE)) != null) { state = State.PROTOCOL; int i = str.indexOf('?'); if (i > 0) { @@ -69,13 +81,13 @@ private void tryDecode(ByteBuffer b) { } return; case PROTOCOL: - if ((str = readUntil(b, CRLF)) != null) { + if ((str = readUntil(CRLF)) != null) { state = State.HEADER; req.setProtocol(str); } return; case HEADER: - if ((str = readUntil(b, CRLF)) != null) { + if ((str = readUntil(CRLF)) != null) { if (str.isEmpty()) { state = State.DONE; return; @@ -91,45 +103,38 @@ private void tryDecode(ByteBuffer b) { } } - private String readUntil(ByteBuffer b, byte c) { - byte[] a = b.array(); - int o = b.position() + b.arrayOffset(); - int l = b.limit() + b.arrayOffset(); - + private String readUntil(byte c) { int i = o; while (i < l) { - if (a[i] == c) { - String str = decoder.append(a, o, i).done(); - b.position(i + 1); + if (buf[i] == c) { + String str = decoder.append(buf, o, i).done(); + o = i + 1; return str; } i++; } - decoder.append(a, o, l); - b.position(l); + decoder.append(buf, o, l); + o = l; return null; } - private String readUntil(ByteBuffer b, byte[] c) { - byte[] a = b.array(); - int base = b.arrayOffset(); - int o = b.position() + base; - int l = b.limit() + base; - + private String readUntil(byte[] c) { int i = o; while (i < l) { - byte t = a[i++]; + byte t = buf[i++]; if (c[f++] != t) { decoder.append(c, 0, f - 1); f = 0; } else if (f == c.length) { + if (i > c.length) + decoder.append(buf, o, i - c.length); f = 0; - b.position(i - base); - return decoder.append(a, o, i - c.length).done(); + o = i; + return decoder.done(); } } - decoder.append(a, o, l - f); - b.position(l); + decoder.append(buf, o, l - f); + o = l; return null; } diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http2/Http2Processor.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http2/Http2Processor.java index 3d461728..fe5e7a02 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http2/Http2Processor.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http2/Http2Processor.java @@ -34,7 +34,7 @@ public class Http2Processor implements NIOConnectionHandler, Http2FlowControl { static final Logger logger = LoggerFactory.getLogger(Http2Processor.class); - public static final byte[] PRI = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.US_ASCII); + public static final ByteBuffer PRI = ByteBuffer.wrap("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.US_ASCII)); public static final int NO_ERROR = 0; public static final int PROTOCOL_ERROR = 1; From b1322290166c4caf07434e958238c781dbd8fc6e Mon Sep 17 00:00:00 2001 From: Unknow Date: Tue, 24 Mar 2026 20:46:40 +0100 Subject: [PATCH 02/31] update chunked encoding --- .../servlet/http11/Http11Processor.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java index 02da76ed..d1f2ffb2 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java @@ -59,19 +59,9 @@ private final void process(ByteBuffer b) throws IOException { decode(b); break; case CONTENT: - if (b.remaining() < contentLength) { - contentLength -= b.remaining(); - ByteBuffer data = ByteBuffer.allocate(b.remaining()); - data.put(b); - dec.addContent(data); - } else { + if (readContent(b)) { state = START; - ByteBuffer data = ByteBuffer.allocate((int) contentLength); - data.put(b.slice().limit((int) contentLength)); - dec.addContent(data); dec.closeContent(); - b.position(b.position() + (int) contentLength); - contentLength = 0; } break; case CHUNKED_START: @@ -98,15 +88,8 @@ private final void process(ByteBuffer b) throws IOException { sb.append((char) c); break; case CHUNKED_DATA: - if (b.remaining() < contentLength) { - contentLength -= b.remaining(); - dec.addContent(b.slice()); - b.position(b.limit()); - } else { - // read CRLF + if (readContent(b)) { state = CHUNKED_END; - dec.addContent(b.slice().limit((int) contentLength)); - b.position(b.position() + (int) contentLength); contentLength = 0; } break; @@ -133,6 +116,22 @@ private final void process(ByteBuffer b) throws IOException { } } + private boolean readContent(ByteBuffer b) { + if (b.remaining() < contentLength) { + contentLength -= b.remaining(); + ByteBuffer data = ByteBuffer.allocate(b.remaining()); + data.put(b); + dec.addContent(data); + return false; + } + ByteBuffer data = ByteBuffer.allocate((int) contentLength); + data.put(b.slice().limit(b.position() + (int) contentLength)); + dec.addContent(data); + b.position(b.position() + (int) contentLength); + contentLength = 0; + return true; + } + @Override public boolean canClose(long now, boolean stop) { return lock.isDone() && co.keepAliveReached(now); From 452f1f43e275f92d68996868affa9d2f52510c59 Mon Sep 17 00:00:00 2001 From: Unknow Date: Tue, 24 Mar 2026 20:50:23 +0100 Subject: [PATCH 03/31] fix content --- .../java/unknow/server/servlet/http11/Http11Processor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java index d1f2ffb2..146262d9 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java @@ -120,13 +120,12 @@ private boolean readContent(ByteBuffer b) { if (b.remaining() < contentLength) { contentLength -= b.remaining(); ByteBuffer data = ByteBuffer.allocate(b.remaining()); - data.put(b); - dec.addContent(data); + dec.addContent(data.put(b).flip()); return false; } ByteBuffer data = ByteBuffer.allocate((int) contentLength); data.put(b.slice().limit(b.position() + (int) contentLength)); - dec.addContent(data); + dec.addContent(data.flip()); b.position(b.position() + (int) contentLength); contentLength = 0; return true; From 227cbb99850370cf261911f1362d1741f78a9c51 Mon Sep 17 00:00:00 2001 From: Unknow Date: Tue, 24 Mar 2026 21:54:45 +0100 Subject: [PATCH 04/31] fix error --- .../main/java/unknow/server/servlet/http11/Http11Processor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java index 146262d9..8a472241 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Processor.java @@ -124,7 +124,7 @@ private boolean readContent(ByteBuffer b) { return false; } ByteBuffer data = ByteBuffer.allocate((int) contentLength); - data.put(b.slice().limit(b.position() + (int) contentLength)); + data.put(b.slice().limit((int) contentLength)); dec.addContent(data.flip()); b.position(b.position() + (int) contentLength); contentLength = 0; From e163bc108e90fca0345df9dcbf5d4124f34ddf67 Mon Sep 17 00:00:00 2001 From: Unknow Date: Tue, 24 Mar 2026 23:42:11 +0100 Subject: [PATCH 05/31] try fix race condition on OP_WRITE --- .../src/main/java/unknow/server/nio/NIOConnection.java | 3 --- .../src/main/java/unknow/server/nio/NIOWorker.java | 9 ++++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index f5fe7902..3829f305 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -100,15 +100,12 @@ public final void write(ByteBuffer buf) throws InterruptedException, IOException pending.put(buf); if (pending.size() > 10) flush(); - else - key.interestOpsOr(SelectionKey.OP_WRITE); } @SuppressWarnings("resource") public final void flush() { if (!hasPendingWrites()) return; - key.interestOpsOr(SelectionKey.OP_WRITE); key.selector().wakeup(); } diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java index 04157814..db5ecbb6 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java @@ -110,14 +110,17 @@ protected void onSelect(long now, boolean close) { private void checkPending(long now, boolean close) { if (head == null) return; - long end = now - 1_000_000_000L; +// long end = now - 1_000_000_000L; NIOConnection co = head; - while (co != null && co.lastCheck < end) { + while (co != null /*&& co.lastCheck < end*/) { NIOConnection next = co.next; if (!co.key.isValid() || co.canClose(now, close)) startClose(co, now); - else + else { + if (co.hasPendingWrites()) + co.key.interestOpsOr(SelectionKey.OP_WRITE); co.lastCheck = now; + } co = next; } From 2bf85cb3fcd7fffbf28c8803a808dd2e84218be0 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 11:45:00 +0100 Subject: [PATCH 06/31] try fix race condition on op_write --- .../java/unknow/server/nio/NIOConnection.java | 30 ++++++++++++++++--- .../java/unknow/server/nio/NIOWorker.java | 9 ++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index 3829f305..2ea7a2f8 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -13,6 +13,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.SSLEngine; @@ -36,6 +37,9 @@ public final class NIOConnection extends NIOHandlerDelegate { protected final SelectionKey key; protected final SocketChannel channel; + private final AtomicBoolean writeScheduled; + private final WriteCheck writeCheck; + private InetSocketAddress local; private InetSocketAddress remote; @@ -59,6 +63,8 @@ public NIOConnection(NIOWorker worker, SelectionKey key, NIOConnectionHandler ha this.worker = worker; this.key = key; this.channel = (SocketChannel) key.channel(); + this.writeScheduled = new AtomicBoolean(false); + this.writeCheck = new WriteCheck(); this.out = new Out(this); this.pending = new LinkedBlockingDeque<>(); this.writes = new ByteBuffers(16); @@ -98,8 +104,8 @@ public final void write(ByteBuffer buf) throws InterruptedException, IOException if (!key.isValid()) throw new IOException("already closed"); pending.put(buf); - if (pending.size() > 10) - flush(); + if (writeScheduled.compareAndSet(false, true)) + execute(writeCheck); } @SuppressWarnings("resource") @@ -167,8 +173,13 @@ protected final void beforeWrite(long now) throws IOException { public final void onWrite(long now) throws IOException { lastAction = now; writes.compact(); - if (!hasPendingWrites()) - key.interestOpsAnd(~SelectionKey.OP_WRITE); + if (!hasPendingWrites()) { + writeScheduled.set(false); + if (!hasPendingWrites()) + key.interestOpsAnd(~SelectionKey.OP_WRITE); + else + writeScheduled.set(true); + } handler.onWrite(now); } @@ -314,4 +325,15 @@ private void writeBuffer() throws IOException { } } } + + private class WriteCheck implements WorkerTask { + + @Override + public void run(long now) { + if (hasPendingWrites()) + key.interestOpsOr(SelectionKey.OP_WRITE); + else + writeScheduled.set(false); + } + } } diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java index db5ecbb6..04157814 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java @@ -110,17 +110,14 @@ protected void onSelect(long now, boolean close) { private void checkPending(long now, boolean close) { if (head == null) return; -// long end = now - 1_000_000_000L; + long end = now - 1_000_000_000L; NIOConnection co = head; - while (co != null /*&& co.lastCheck < end*/) { + while (co != null && co.lastCheck < end) { NIOConnection next = co.next; if (!co.key.isValid() || co.canClose(now, close)) startClose(co, now); - else { - if (co.hasPendingWrites()) - co.key.interestOpsOr(SelectionKey.OP_WRITE); + else co.lastCheck = now; - } co = next; } From 70cae40ddbfdcc677245b6442ce77ec092792225 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 13:38:07 +0100 Subject: [PATCH 07/31] test without task --- .../java/unknow/server/nio/NIOConnection.java | 5 ++++- .../test/java/unknow/server/servlet/Test.java | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 unknow-server-servlet/src/test/java/unknow/server/servlet/Test.java diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index 2ea7a2f8..e688b6da 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -105,7 +105,10 @@ public final void write(ByteBuffer buf) throws InterruptedException, IOException throw new IOException("already closed"); pending.put(buf); if (writeScheduled.compareAndSet(false, true)) - execute(writeCheck); + key.interestOpsOr(SelectionKey.OP_WRITE); + if (pending.size() > 10) + flush(); +// execute(writeCheck); } @SuppressWarnings("resource") diff --git a/unknow-server-servlet/src/test/java/unknow/server/servlet/Test.java b/unknow-server-servlet/src/test/java/unknow/server/servlet/Test.java new file mode 100644 index 00000000..50db0881 --- /dev/null +++ b/unknow-server-servlet/src/test/java/unknow/server/servlet/Test.java @@ -0,0 +1,16 @@ +package unknow.server.servlet; + +import java.nio.ByteBuffer; + +import unknow.server.servlet.http2.Http2Processor; + +public class Test { + public static void main(String[] arg) { + ByteBuffer b = ByteBuffer.allocate(1024); + b.put(Http2Processor.PRI.array()); + b.put(new byte[] { 0, 0, 0 }); + b.flip(); + System.out.println(Http2Processor.PRI.remaining()); + System.out.println(Http2Processor.PRI.mismatch(b)); + } +} From 714f294c1d98ac5aef1e4f16d8beeaf68f090bbd Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 16:31:39 +0100 Subject: [PATCH 08/31] up --- .../java/unknow/server/nio/NIOConnection.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index e688b6da..6ac468b2 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -105,17 +105,14 @@ public final void write(ByteBuffer buf) throws InterruptedException, IOException throw new IOException("already closed"); pending.put(buf); if (writeScheduled.compareAndSet(false, true)) - key.interestOpsOr(SelectionKey.OP_WRITE); - if (pending.size() > 10) - flush(); -// execute(writeCheck); + execute(writeCheck); } @SuppressWarnings("resource") public final void flush() { - if (!hasPendingWrites()) - return; - key.selector().wakeup(); +// if (!hasPendingWrites()) +// return; +// key.selector().wakeup(); } /** @@ -180,8 +177,8 @@ public final void onWrite(long now) throws IOException { writeScheduled.set(false); if (!hasPendingWrites()) key.interestOpsAnd(~SelectionKey.OP_WRITE); - else - writeScheduled.set(true); + else if (writeScheduled.compareAndSet(false, true)) + key.interestOpsOr(SelectionKey.OP_WRITE); } handler.onWrite(now); } @@ -214,7 +211,7 @@ public final void doneClosing() { @Override public String toString() { - return getClass() + "[local=" + getLocal() + " remote=" + getRemote() + "]"; + return getClass() + "[local=" + getLocal() + " remote=" + getRemote() + "] writes: " + hasPendingWrites(); } /** output stream for this connection */ @@ -330,7 +327,6 @@ private void writeBuffer() throws IOException { } private class WriteCheck implements WorkerTask { - @Override public void run(long now) { if (hasPendingWrites()) From 7f3bf240d06fb5b9281dbb20db7a8d4a9c92b564 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 20:12:38 +0100 Subject: [PATCH 09/31] cleanup write process & ssl --- .../java/unknow/server/nio/NIOConnection.java | 25 ++++----- .../server/nio/NIOConnectionHandler.java | 20 ++----- .../unknow/server/nio/NIOHandlerDelegate.java | 12 ++--- .../java/unknow/server/nio/NIOSSLHandler.java | 54 +++++++++---------- .../server/servlet/http11/Http11Worker.java | 9 +++- 5 files changed, 54 insertions(+), 66 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index 6ac468b2..d92999b7 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -10,9 +10,9 @@ import java.nio.channels.SelectionKey; import java.nio.channels.SocketChannel; import java.util.Arrays; -import java.util.concurrent.BlockingQueue; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicBoolean; import javax.net.ssl.SSLEngine; @@ -43,7 +43,7 @@ public final class NIOConnection extends NIOHandlerDelegate { private InetSocketAddress local; private InetSocketAddress remote; - final BlockingQueue pending; + final Queue pending; final ByteBuffers writes; long lastCheck; @@ -66,7 +66,7 @@ public NIOConnection(NIOWorker worker, SelectionKey key, NIOConnectionHandler ha this.writeScheduled = new AtomicBoolean(false); this.writeCheck = new WriteCheck(); this.out = new Out(this); - this.pending = new LinkedBlockingDeque<>(); + this.pending = new ConcurrentLinkedQueue<>(); this.writes = new ByteBuffers(16); } @@ -103,12 +103,11 @@ public void init(NIOConnection co, long now, SSLEngine sslEngine) throws IOExcep public final void write(ByteBuffer buf) throws InterruptedException, IOException { if (!key.isValid()) throw new IOException("already closed"); - pending.put(buf); + pending.offer(buf); if (writeScheduled.compareAndSet(false, true)) execute(writeCheck); } - @SuppressWarnings("resource") public final void flush() { // if (!hasPendingWrites()) // return; @@ -164,9 +163,10 @@ public boolean isClosed() { } protected final void beforeWrite(long now) throws IOException { - while (writes.len < 16 && !pending.isEmpty()) - handler.prepareWrite(pending.poll(), now, writes); - handler.beforeWrite(now, writes); + ByteBuffer b; + while ((b = pending.poll()) != null) + writes.accept(b); + handler.transformWrite(writes, now); } @Override @@ -216,13 +216,14 @@ public String toString() { /** output stream for this connection */ public static final class Out extends OutputStream { + private static final int BUF_SIZE = 8192; private NIOConnection h; private ByteBuffer buf; private Out(NIOConnection h) { this.h = h; - this.buf = ByteBuffer.allocate(4096); + this.buf = ByteBuffer.allocate(BUF_SIZE); } @Override @@ -254,7 +255,7 @@ public synchronized void write(byte[] b, int off, int len) throws IOException { len -= r; off += r; writeBuffer(); - if (len < 4096) + if (len < BUF_SIZE) buf.put(b, off, len); else try { @@ -318,7 +319,7 @@ private void writeBuffer() throws IOException { return; try { h.write(buf.flip()); - buf = ByteBuffer.allocate(4096); + buf = ByteBuffer.allocate(BUF_SIZE); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException(e); diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java index 447c18b8..96934b00 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java @@ -6,6 +6,8 @@ import javax.net.ssl.SSLEngine; +import unknow.server.util.io.ByteBuffers; + @SuppressWarnings("unused") public interface NIOConnectionHandler { @@ -48,25 +50,13 @@ default void onRead(ByteBuffer b, long now) throws IOException { // ok } /** - * called before a buffer is written (allow to collect buffers) - * - * @param b buffer to be written - * @param now nanoTime - * @param c consumer of generated buffers - * @throws IOException on io exception - */ - default void prepareWrite(ByteBuffer b, long now, Consumer c) throws IOException { - c.accept(b); - } - - /** - * called before a buffer is written + * called before some buffers are written * + * @param buffers buffers to be written * @param now nanoTime - * @param c consumer of generated buffers * @throws IOException on io exception */ - default void beforeWrite(long now, Consumer c) throws IOException { // ok + default void transformWrite(ByteBuffers buffers, long now) throws IOException { // ok } /** diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOHandlerDelegate.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOHandlerDelegate.java index a3a43fb9..7331d0a9 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOHandlerDelegate.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOHandlerDelegate.java @@ -2,10 +2,11 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.function.Consumer; import javax.net.ssl.SSLEngine; +import unknow.server.util.io.ByteBuffers; + public class NIOHandlerDelegate implements NIOConnectionHandler { protected final NIOConnectionHandler handler; @@ -34,13 +35,8 @@ public void onRead(ByteBuffer b, long now) throws IOException { } @Override - public void prepareWrite(ByteBuffer b, long now, Consumer c) throws IOException { - handler.prepareWrite(b, now, c); - } - - @Override - public void beforeWrite(long now, Consumer c) throws IOException { - handler.beforeWrite(now, c); + public void transformWrite(ByteBuffers buffers, long now) throws IOException { + handler.transformWrite(buffers, now); } @Override diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java index 81e74440..b3fd0f82 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java @@ -4,7 +4,6 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; @@ -25,20 +24,20 @@ public class NIOSSLHandler extends NIOHandlerDelegate { private static final ByteBuffer EMPTY = ByteBuffer.allocate(0); private final SSLContext sslContext; + private final ByteBuffers rawOut; private NIOConnection co; private SSLEngine sslEngine; private ByteBuffer rawIn; - private ByteBuffers rawOut; - private ByteBuffers app; + private ByteBuffer app; private boolean handshake; private int packetBufferSize; - private int applicationBufferSize; public NIOSSLHandler(SSLContext sslContext, NIOConnectionHandler handler) { super(handler); this.sslContext = sslContext; + this.rawOut = new ByteBuffers(16); } @Override @@ -52,13 +51,8 @@ public void init(NIOConnection co, long now, SSLEngine e) throws IOException { this.co = co; this.sslEngine = sslContext.createSSLEngine(remote.getHostString(), remote.getPort()); this.packetBufferSize = sslEngine.getSession().getPacketBufferSize(); - this.applicationBufferSize = sslEngine.getSession().getApplicationBufferSize(); this.rawIn = ByteBuffer.allocate(packetBufferSize); - this.rawOut = new ByteBuffers(16); - this.app = new ByteBuffers(16); - for (int i = 0; i < applicationBufferSize; i += 4096) - app.accept(ByteBuffer.allocate(4096)); - + this.app = ByteBuffer.allocate(sslEngine.getSession().getApplicationBufferSize()); this.handshake = true; handler.init(co, now, sslEngine); @@ -90,40 +84,34 @@ public void onRead(ByteBuffer b, long now) throws IOException { if (handshake) { rawIn.flip(); - SSLEngineResult r = sslEngine.unwrap(rawIn, app.buf, 0, app.len); - logger.trace("unwrap {}", r); + SSLEngineResult r = sslEngine.unwrap(rawIn, app); + logger.debug("unwrap {}", r); rawIn.compact(); if (checkHandshake(r.getHandshakeStatus(), now)) return; - app.drain(buf -> handler.onRead(buf, now)); + handler.onRead(app.flip(), now); + app.clear(); } rawIn.flip(); while (rawIn.hasRemaining()) { - SSLEngineResult r = sslEngine.unwrap(rawIn, app.buf, 0, app.len); - logger.trace("unwrap {}", r); + SSLEngineResult r = sslEngine.unwrap(rawIn, app); + logger.debug("unwrap {}", r); if (r.getStatus() == Status.BUFFER_UNDERFLOW || r.getStatus() == Status.CLOSED) break; - if (r.getStatus() == SSLEngineResult.Status.BUFFER_OVERFLOW) { - for (int i = app.remaining(); i < applicationBufferSize; i += 4096) - app.accept(ByteBuffer.allocate(4096)); - } else - app.drain(buf -> handler.onRead(buf, now)); + handler.onRead(app.flip(), now); + app.clear(); } rawIn.compact(); } @Override - public void prepareWrite(ByteBuffer b, long now, Consumer c) throws IOException { - handler.prepareWrite(b, now, rawOut); - } - - @Override - public void beforeWrite(long now, Consumer c) throws IOException { + public void transformWrite(ByteBuffers buffers, long now) throws IOException { + rawOut.accept(buffers); while (!rawOut.isEmpty()) { ByteBuffer out = ByteBuffer.allocate(packetBufferSize); SSLEngineResult r = sslEngine.wrap(rawOut.buf, 0, rawOut.len, out); - logger.trace("wrap {}", r); - c.accept(out.flip()); + logger.debug("wrap {}", r); + buffers.accept(out.flip()); if (r.getStatus() == Status.CLOSED) { AtomicInteger l = new AtomicInteger(0); rawOut.drain(b -> l.getAndAdd(b.remaining())); @@ -131,8 +119,14 @@ public void beforeWrite(long now, Consumer c) throws IOException { logger.warn("{} remaining data {}", co, l); break; } - rawOut.compact(); + while (r.getHandshakeStatus() == HandshakeStatus.NEED_WRAP) { + out = ByteBuffer.allocate(packetBufferSize); + r = sslEngine.wrap(rawOut.buf, 0, rawOut.len, out); + buffers.accept(out.flip()); + logger.debug("wrap {}", r); + } checkHandshake(r.getHandshakeStatus(), now); + rawOut.compact(); } } @@ -169,7 +163,7 @@ private boolean checkHandshake(HandshakeStatus hs, long now) throws IOException case NEED_UNWRAP: case NEED_UNWRAP_AGAIN: rawIn.flip(); - r = sslEngine.unwrap(rawIn, app.buf, 0, app.len); + r = sslEngine.unwrap(rawIn, app); logger.trace("unwrap {}", r); rawIn.compact(); if (r.getStatus() == Status.BUFFER_UNDERFLOW) diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Worker.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Worker.java index e1678ee3..5a346b1c 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Worker.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/Http11Worker.java @@ -4,6 +4,9 @@ import java.nio.charset.StandardCharsets; import java.util.Collection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import jakarta.servlet.http.Cookie; import unknow.server.nio.NIOConnection.Out; import unknow.server.servlet.HttpError; @@ -13,6 +16,7 @@ /** http/1.1 worker */ public final class Http11Worker extends HttpWorker { + private static final Logger logger = LoggerFactory.getLogger(Http11Worker.class); private static final byte[] CRLF = { '\r', '\n' }; private static final byte[] QUOTE = new byte[] { '\\', '"' }; @@ -102,6 +106,7 @@ else if (rawStream.isChunked()) { @SuppressWarnings("resource") @Override public final boolean doStart() throws IOException, InterruptedException { + logger.debug("{} doStart", co); if ("100-continue".equals(req.getHeader("expect"))) { Out out = co.getOut(); out.write(HttpError.CONTINUE.encoded); @@ -122,7 +127,9 @@ public final boolean doStart() throws IOException, InterruptedException { @Override protected void doDone() throws IOException { - if (!"keep-alive".equalsIgnoreCase(res.getHeader("connection"))) + String header = res.getHeader("connection"); + logger.debug("{} doDone {}", co, header); + if (!"keep-alive".equalsIgnoreCase(header)) co.getOut().close(); else co.flush(); From f72e4a0f8556be8e4a9b47066c30c88ba9c3666f Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 20:13:51 +0100 Subject: [PATCH 10/31] missing update --- .../unknow/server/util/io/ByteBuffers.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java index c1924055..7a79715e 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java @@ -15,13 +15,26 @@ public ByteBuffers(int l) { len = 0; } + private void ensureCapacity(int l) { + if (l > buf.length) + buf = Arrays.copyOf(buf, l); + } + @Override public void accept(ByteBuffer b) { - if (len == buf.length) - buf = Arrays.copyOf(buf, len + 1); + ensureCapacity(len + 1); buf[len++] = b; } + public void accept(ByteBuffers buffers) { + if (buffers.isEmpty()) + return; + ensureCapacity(len + buf.length); + System.arraycopy(buffers.buf, 0, buf, len, buffers.len); + len += buffers.len; + buffers.clear(); + } + public boolean isEmpty() { return len == 0; } @@ -72,4 +85,31 @@ public void drain(ConsumerWithException c) } len = 0; } + + /** + * drain buffers + * @param exception thrown from consumer + * @param c consumer of buffer + * @throws E from Consumer + */ + public void drainNonEmpty(ConsumerWithException c) throws E { + if (len == 0) + return; + + int i = 0; + while (i < len) { + ByteBuffer b = buf[i]; + if (b.position() == 0 && b.limit() == b.capacity()) + break; + c.accept(b.flip()); + i++; + } + if (i == 0) + return; + int l = len - 1; + len -= i; + System.arraycopy(buf, i, buf, 0, len); + while (l >= len) + buf[l] = null; + } } From 4c2eb17bc0ab9eb29604ada44464a0676f4a90cd Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 22:48:02 +0100 Subject: [PATCH 11/31] fix ssl closing --- .../src/main/java/unknow/server/nio/NIOSSLHandler.java | 5 +++-- .../src/main/java/unknow/server/nio/NIOWorker.java | 7 ++++--- .../main/java/unknow/server/servlet/HttpConnection.java | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java index b3fd0f82..35d5e4c5 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java @@ -128,6 +128,7 @@ public void transformWrite(ByteBuffers buffers, long now) throws IOException { checkHandshake(r.getHandshakeStatus(), now); rawOut.compact(); } + handler.transformWrite(buffers, now); } @Override @@ -137,10 +138,10 @@ public boolean hasPendingWrites() { @Override public boolean finishClosing(long now) { - if (!handler.finishClosing(now)) - return false; if (sslEngine.isOutboundDone()) return true; + if (!handler.finishClosing(now)) + return false; sslEngine.closeOutbound(); try { diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java index 04157814..f00f8f58 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java @@ -141,7 +141,7 @@ private void finishClosing(long now) { Iterator it = closing.iterator(); while (it.hasNext()) { NIOConnection co = it.next(); - if (!co.key.isValid() || !co.hasPendingWrites() && co.finishClosing(now) || closingTimeout(co, now)) { + if (!co.key.isValid() || co.finishClosing(now) || closingTimeout(co, now)) { it.remove(); doneClose(co); } @@ -211,9 +211,10 @@ private void doRead(NIOConnection co, long now) throws IOException { } private void doWrite(NIOConnection co, long now) throws IOException { - co.beforeWrite(now); + if (co.writes.isEmpty()) + co.beforeWrite(now); long l = co.channel.write(co.writes.buf, 0, co.writes.len); - logger.trace("{} writen {}", co, l); + logger.trace("{} writen {} {}", co, l, co.key); if (l > 0) { co.onWrite(now); if (co.isClosed()) diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java index 1d9f1287..c7148fc8 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java @@ -119,7 +119,7 @@ public void startClose(long now) { @Override public boolean finishClosing(long now) { - return p == null || p.finishClosing(now); + return co.hasPendingWrites() || p == null || p.finishClosing(now); } @Override From d3e36023a466b08e008515d0e5b36d56d0d7f156 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 26 Mar 2026 23:55:41 +0100 Subject: [PATCH 12/31] increase some buffer size & add bytebufferreader --- .../java/unknow/server/nio/NIOConnection.java | 2 +- .../java/unknow/server/nio/NIOWorker.java | 2 +- .../servlet/impl/AbstractServletOutput.java | 2 +- .../servlet/impl/ServletRequestImpl.java | 21 ++++--- .../server/util/io/ByteBufferInputStream.java | 6 +- .../server/util/io/ByteBufferReader.java | 62 +++++++++++++++++++ 6 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index d92999b7..075a0167 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -216,7 +216,7 @@ public String toString() { /** output stream for this connection */ public static final class Out extends OutputStream { - private static final int BUF_SIZE = 8192; + private static final int BUF_SIZE = 16 * 1024; private NIOConnection h; private ByteBuffer buf; diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java index f00f8f58..49a2ba65 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java @@ -33,7 +33,7 @@ public final class NIOWorker extends NIOLoop implements NIOWorkers { private static final Logger logger = LoggerFactory.getLogger(NIOWorker.class); - private static final int BUF_LEN = 16000; + private static final int BUF_LEN = 64 * 1024; /** executor for delegating task */ private final ExecutorService executor; diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/AbstractServletOutput.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/AbstractServletOutput.java index 8e5f8433..bbfb7f4d 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/AbstractServletOutput.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/AbstractServletOutput.java @@ -29,7 +29,7 @@ protected AbstractServletOutput(ServletResponseImpl res, int position) { this.res = res; this.position = position; if (res != null) { - this.bufferSize = Math.max(res.getBufferSize(), 4096); + this.bufferSize = Math.max(res.getBufferSize(), 16 * 1024); this.buffer = ByteBuffer.allocate(bufferSize + position).position(position); } else { this.buffer = null; diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java index cbd499fa..4bb3d75b 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java @@ -49,6 +49,8 @@ import unknow.server.servlet.impl.session.SessionFactory; import unknow.server.servlet.utils.PathUtils; import unknow.server.util.data.ArrayMap; +import unknow.server.util.io.ByteBufferInputStream; +import unknow.server.util.io.ByteBufferReader; /** * @author unknow @@ -62,7 +64,7 @@ public class ServletRequestImpl implements HttpServletRequest { protected final HttpConnection co; private final DispatcherType type; - private final ServletInputStreamImpl input; + private final ByteBufferInputStream input; private String requestUri; private int pathInfoIndex; @@ -92,6 +94,7 @@ public class ServletRequestImpl implements HttpServletRequest { private List locales; private BufferedReader reader; + private ServletInputStreamImpl stream; /** * create new ServletRequestImpl @@ -102,13 +105,13 @@ public class ServletRequestImpl implements HttpServletRequest { public ServletRequestImpl(HttpConnection co, DispatcherType type) { this.co = co; this.type = type; - this.input = new ServletInputStreamImpl(); + this.input = new ByteBufferInputStream(); this.headers = new HashMap<>(); } public void append(ByteBuffer buf) { - input.append(buf); + input.addBuffer(buf); } public void close() { @@ -116,7 +119,7 @@ public void close() { } public boolean isClosed() { - return input.isFinished(); + return input.isClosed(); } public void setMethod(String method) { @@ -193,7 +196,7 @@ public void setServletPath(String servletPath) { } public void clearInput() throws IOException { - while (!input.isFinished()) + while (!input.isClosed()) input.skip(Long.MAX_VALUE); } @@ -327,14 +330,18 @@ public long getContentLengthLong() { public ServletInputStream getInputStream() throws IOException { if (reader != null) throw new IllegalStateException("getReader() called"); - return input; + if (stream == null) + stream = new ServletInputStreamImpl(input); + return stream; } @Override public BufferedReader getReader() throws IOException { + if (stream != null) + throw new IllegalStateException("getInputStream() called"); if (reader != null) return reader; - return reader = new BufferedReader(new InputStreamReader(input, getCharacterEncoding())); + return reader = new BufferedReader(new ByteBufferReader(input, getCharacterEncoding())); } @Override diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferInputStream.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferInputStream.java index cfffd405..fd6be997 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferInputStream.java +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferInputStream.java @@ -30,13 +30,17 @@ public void drain(Collection list) { buffers.drainTo(list); } + public boolean isOef() throws IOException { + return buffer() == EOF; + } + /** * get next buffer to read (wait if no buffer available) * @return * @throws InterruptedException * @throws IOException */ - private final ByteBuffer buffer() throws IOException { + final ByteBuffer buffer() throws IOException { if (current == null || current != EOF && !current.hasRemaining()) { try { current = buffers.take(); diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java new file mode 100644 index 00000000..a919ee4a --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java @@ -0,0 +1,62 @@ +package unknow.server.util.io; + +import java.io.IOException; +import java.io.Reader; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; + +public class ByteBufferReader extends Reader { + private final ByteBufferInputStream in; + private final CharsetDecoder decoder; + private final CharBuffer buf; + + public ByteBufferReader(ByteBufferInputStream in, String charset) { + this(in, Charset.forName(charset).newDecoder()); + } + + public ByteBufferReader(ByteBufferInputStream in, Charset charset) { + this(in, charset.newDecoder()); + } + + public ByteBufferReader(ByteBufferInputStream in, CharsetDecoder decoder) { + this.in = in; + this.decoder = decoder; + buf = CharBuffer.allocate(8192); + } + + private void decode() throws IOException { + if (buf.hasRemaining() || in.isOef()) + return; + buf.compact(); + CoderResult r = decoder.decode(in.buffer(), buf, in.hasRemaining() && in.isClosed()); + buf.flip(); + if (r.isError()) + r.throwException(); + } + + @Override + public int read() throws IOException { + if (in.isOef()) + return -1; + decode(); + return buf.get(); + } + + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + if (in.isOef()) + return -1; + decode(); + len = Math.min(len, buf.remaining()); + buf.get(cbuf, off, len); + return len; + } + + @Override + public void close() throws IOException { + in.close(); + } + +} From 67fe5a38f9d38039d56b7bfad9f25a665b6644f5 Mon Sep 17 00:00:00 2001 From: Unknow Date: Fri, 27 Mar 2026 09:01:06 +0100 Subject: [PATCH 13/31] fix byteBuffers --- .../src/main/java/unknow/server/util/io/ByteBuffers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java index 7a79715e..6ee966a4 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBuffers.java @@ -29,7 +29,7 @@ public void accept(ByteBuffer b) { public void accept(ByteBuffers buffers) { if (buffers.isEmpty()) return; - ensureCapacity(len + buf.length); + ensureCapacity(len + buffers.len); System.arraycopy(buffers.buf, 0, buf, len, buffers.len); len += buffers.len; buffers.clear(); From e66baab737b1fa688345f07530d56d499915b7dc Mon Sep 17 00:00:00 2001 From: Unknow Date: Fri, 27 Mar 2026 20:00:49 +0100 Subject: [PATCH 14/31] add some specialized decoder --- .../server/servlet/http11/RequestDecoder.java | 2 +- .../servlet/impl/ServletRequestImpl.java | 19 +-- .../server/servlet/utils/UrlDecoder.java | 150 ++++++++++++++++++ .../main/java/unknow/server/util/Decoder.java | 23 +++ .../unknow/server/util/DefaultDecoder.java | 21 +++ .../java/unknow/server/util/Utf8Decoder.java | 64 ++++++++ .../server/util/io/ByteBufferCache.java | 67 ++++++++ .../server/util/io/ByteBufferReader.java | 20 +-- 8 files changed, 338 insertions(+), 28 deletions(-) create mode 100644 unknow-server-servlet/src/main/java/unknow/server/servlet/utils/UrlDecoder.java create mode 100644 unknow-server-util/src/main/java/unknow/server/util/Decoder.java create mode 100644 unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java create mode 100644 unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java create mode 100644 unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferCache.java diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java index 1a94c581..8254b3f1 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/http11/RequestDecoder.java @@ -26,7 +26,7 @@ private enum State { public RequestDecoder(Http11Processor co) { this.co = co; - decoder = new Utf8Decoder(new StringBuilder()); + decoder = new Utf8Decoder(); buf = new byte[4096]; reset(); } diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java index 4bb3d75b..8f576485 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletRequestImpl.java @@ -5,7 +5,6 @@ import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; import java.io.Reader; import java.io.StringReader; import java.io.UnsupportedEncodingException; @@ -48,6 +47,7 @@ import unknow.server.servlet.HttpConnection; import unknow.server.servlet.impl.session.SessionFactory; import unknow.server.servlet.utils.PathUtils; +import unknow.server.servlet.utils.UrlDecoder; import unknow.server.util.data.ArrayMap; import unknow.server.util.io.ByteBufferInputStream; import unknow.server.util.io.ByteBufferReader; @@ -164,8 +164,10 @@ private void parseParam() { } try { - if ("POST".equals(getMethod()) && "application/x-www-form-urlencoded".equalsIgnoreCase(getContentType())) - parseContentParam(map); + if ("POST".equals(getMethod()) && "application/x-www-form-urlencoded".equalsIgnoreCase(getContentType())) { + new UrlDecoder(input).process(map); + contentLength = 0; + } } catch (IOException e) { logger.error("failed to parse params from content", e); } @@ -175,17 +177,6 @@ private void parseParam() { parameter.put(e.getKey(), e.getValue().toArray(s)); } - /** - * @param p - * @throws IOException - */ - private void parseContentParam(Map> p) throws IOException { - try (BufferedReader r = new BufferedReader(new InputStreamReader(input, getCharacterEncoding()))) { - PathUtils.pathQuery(r, p); - } - contentLength = 0; - } - /** * @param servletPath the servletPath to set */ diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/UrlDecoder.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/UrlDecoder.java new file mode 100644 index 00000000..974f8c96 --- /dev/null +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/UrlDecoder.java @@ -0,0 +1,150 @@ +package unknow.server.servlet.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class UrlDecoder { + private static final byte EQ = '='; + private static final byte AM = '&'; + private static final byte SP = ' '; + private static final int[] HEX = new int[256]; + + static { + Arrays.fill(HEX, -1); + for (int i = '0'; i <= '9'; i++) + HEX[i] = i - '0'; + for (int i = 'A'; i <= 'F'; i++) + HEX[i] = 10 + i - 'A'; + for (int i = 'a'; i <= 'f'; i++) + HEX[i] = 10 + i - 'a'; + } + + private final InputStream in; + private final byte[] buf; + private int off; + private int len; + + private byte[] cbuf; + private int coff; + + private int pctState; // 0 normal, 1 = %, 2 = %X + private int pctHi; + + private String key; + + public UrlDecoder(InputStream in) { + this.in = in; + this.buf = new byte[4096]; + this.cbuf = new byte[4096]; + } + + public void process(Map> map) throws IOException { + while ((len = in.read(buf)) != -1) { + off = 0; + parse(map); + } + if (coff > 0) { + if (key == null) + key = string(); + add(map, string()); + } + } + + private void parse(Map> map) { + while (off < len) { + if (key == null) { + key = decode(); + if (key == null) + continue; + if (buf[off - 1] == AM) { + add(map, ""); + continue; + } + } + + String v = decode(); + if (v != null) + add(map, v); + + } + } + + private void add(Map> map, String value) { + List list = map.get(key); + if (list == null) + map.put(key, list = new ArrayList<>(1)); + list.add(value); + key = null; + } + + private String decode() { + int s = off; + while (off < len) { + byte c = buf[off++]; + if (pctState == 0 && c != '%' && c != '+' && c != EQ && c != AM) + continue; + + append(s); + if (pctState == 1) { + pctHi = HEX[c & 0xFF]; + if (pctHi < 0) + throw new IllegalArgumentException(); + pctState = 2; + continue; + } + if (pctState == 2) { + int lo = HEX[c & 0xFF]; + if (lo < 0) + throw new IllegalArgumentException(); + int val = (pctHi << 4) | lo; + append((byte) val); + pctState = 0; + continue; + } + if (c == EQ || c == AM) + return string(); + + if (c == '%') { + pctState = 1; + continue; + } + if (c == '+') + append(SP); + } + append(s); + return null; + + } + + private String string() { + if (coff == 0) + return ""; + String s = new String(cbuf, 0, coff, StandardCharsets.UTF_8); + coff = 0; + return s; + } + + private void capacity(int l) { + while (cbuf.length < l) + cbuf = Arrays.copyOf(cbuf, cbuf.length * 2); + } + + private void append(int s) { + int l = off - s; + if (l == 0) + return; + capacity(coff + l); + System.arraycopy(buf, s, cbuf, coff, l); + coff += l; + } + + private void append(byte c) { + capacity(coff + 1); + cbuf[coff++] = c; + } +} diff --git a/unknow-server-util/src/main/java/unknow/server/util/Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Decoder.java new file mode 100644 index 00000000..6c001036 --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/Decoder.java @@ -0,0 +1,23 @@ +package unknow.server.util; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +public interface Decoder { + + public static Decoder from(Charset charset) { + if (charset.equals(StandardCharsets.UTF_8)) + return new Utf8Decoder(); + return new DefaultDecoder(charset.newDecoder()); + } + + /** + * append one byte + * @param bbuf byte to append + * @param cbuf produced data + * @param endOfInput the data is done + */ + void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput); +} diff --git a/unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java b/unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java new file mode 100644 index 00000000..dde42bba --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java @@ -0,0 +1,21 @@ +package unknow.server.util; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; + +public class DefaultDecoder implements Decoder { + private final CharsetDecoder dec; + + public DefaultDecoder(CharsetDecoder dec) { + this.dec = dec; + } + + @Override + public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { + CoderResult r = dec.decode(bbuf, cbuf, endOfInput); + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } +} diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java new file mode 100644 index 00000000..1ded1f20 --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -0,0 +1,64 @@ +package unknow.server.util; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; + +public class Utf8Decoder implements Decoder { + /** code point in building */ + private int codePoint; + /** remaining octet to read */ + private int r; + + @Override + public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { + if (bbuf.hasArray()) { + byte[] array = bbuf.array(); + int e = bbuf.limit() + bbuf.arrayOffset(); + int i = bbuf.position() + bbuf.arrayOffset(); + while (i < e && cbuf.remaining() > 1) + append(array[i++] & 0xFF, cbuf); + bbuf.position(i - bbuf.arrayOffset()); + } else { + while (bbuf.hasRemaining() && cbuf.remaining() > 1) { + append(bbuf.get() & 0xFF, cbuf); + } + } + if (endOfInput && !bbuf.hasRemaining() && r > 0) + throw new IllegalArgumentException("Invalid UTF-8 string, truncated continuation bytes"); + } + + private void append(int b, CharBuffer cbuf) { + if (r > 0) { + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + + codePoint = (codePoint << 6) | (b & 0x3F); + if (--r == 0) + append(cbuf); + return; + } + + if ((b >> 7) == 0) // 1-byte ASCII + cbuf.put((char) b); + else if ((b >> 5) == 0b110) { + r = 1; + codePoint = b & 0x1F; + } else if ((b >> 4) == 0b1110) { + r = 2; + codePoint = b & 0x0F; + } else if ((b >> 3) == 0b11110) { + r = 3; + codePoint = b & 0x07; + } else + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + } + + private void append(CharBuffer cbuf) { + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + cbuf.put((char) ((cp >> 10) + 0xD800)).put((char) ((cp & 0x3FF) + 0xDC00)); + } else + cbuf.put((char) codePoint); + codePoint = 0; + } +} diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferCache.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferCache.java new file mode 100644 index 00000000..39942d06 --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferCache.java @@ -0,0 +1,67 @@ +package unknow.server.util.io; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +public class ByteBufferCache { + private static final Map CACHES = new HashMap<>(); + + private final int size; + private final int max; + private final ConcurrentLinkedQueue idle; + private final AtomicInteger count; + + private ByteBufferCache(int size, int max) { + this.size = size; + this.max = max; + this.idle = new ConcurrentLinkedQueue<>(); + this.count = new AtomicInteger(0); + } + + public static void createCache(int size, int max) { + CACHES.computeIfAbsent(size, k -> new ByteBufferCache(size, max)); + } + + public static ByteBufferCached get(int size) { + ByteBufferCache b = CACHES.get(size); + if (b != null) + return b.get(); + return new ByteBufferCached(null, ByteBuffer.allocate(size)); + } + + public ByteBufferCached get() { + ByteBuffer poll = idle.poll(); + if (poll == null) + poll = ByteBuffer.allocate(size); + return new ByteBufferCached(this, poll); + } + + private void free(ByteBuffer b) { + b.clear(); + if (count.getAndUpdate(x -> x == max ? x : x + 1) < max) + idle.offer(b); + } + + public static class ByteBufferCached { + private final ByteBufferCache cache; + private ByteBuffer buf; + + private ByteBufferCached(ByteBufferCache cache, ByteBuffer buf) { + this.cache = cache; + this.buf = buf; + } + + public ByteBuffer buffer() { + return buf; + } + + public void free() { + if (cache != null) + cache.free(buf); + buf = null; + } + } +} diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java index a919ee4a..6ef8ec71 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java @@ -4,36 +4,30 @@ import java.io.Reader; import java.nio.CharBuffer; import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; + +import unknow.server.util.Decoder; public class ByteBufferReader extends Reader { private final ByteBufferInputStream in; - private final CharsetDecoder decoder; + private final Decoder decoder; private final CharBuffer buf; public ByteBufferReader(ByteBufferInputStream in, String charset) { - this(in, Charset.forName(charset).newDecoder()); + this(in, Charset.forName(charset)); } public ByteBufferReader(ByteBufferInputStream in, Charset charset) { - this(in, charset.newDecoder()); - } - - public ByteBufferReader(ByteBufferInputStream in, CharsetDecoder decoder) { this.in = in; - this.decoder = decoder; - buf = CharBuffer.allocate(8192); + this.decoder = Decoder.from(charset); + buf = CharBuffer.allocate(8192).flip(); } private void decode() throws IOException { if (buf.hasRemaining() || in.isOef()) return; buf.compact(); - CoderResult r = decoder.decode(in.buffer(), buf, in.hasRemaining() && in.isClosed()); + decoder.decode(in.buffer(), buf, in.hasRemaining() && in.isClosed()); buf.flip(); - if (r.isError()) - r.throwException(); } @Override From a6ca3de7319de89e99116bf6a2762e899707d200 Mon Sep 17 00:00:00 2001 From: Unknow Date: Fri, 27 Mar 2026 22:01:50 +0100 Subject: [PATCH 15/31] improve utf8Decoder --- .../java/unknow/server/util/Utf8Decoder.java | 131 +++++++++++++++--- 1 file changed, 111 insertions(+), 20 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 1ded1f20..aacf7f48 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -11,35 +11,83 @@ public class Utf8Decoder implements Decoder { @Override public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { - if (bbuf.hasArray()) { - byte[] array = bbuf.array(); - int e = bbuf.limit() + bbuf.arrayOffset(); - int i = bbuf.position() + bbuf.arrayOffset(); - while (i < e && cbuf.remaining() > 1) - append(array[i++] & 0xFF, cbuf); - bbuf.position(i - bbuf.arrayOffset()); - } else { - while (bbuf.hasRemaining() && cbuf.remaining() > 1) { - append(bbuf.get() & 0xFF, cbuf); - } + if (bbuf.hasArray() && cbuf.hasArray()) + fastDecode(bbuf, cbuf); + else { + slowDecode(bbuf, cbuf); } if (endOfInput && !bbuf.hasRemaining() && r > 0) throw new IllegalArgumentException("Invalid UTF-8 string, truncated continuation bytes"); } - private void append(int b, CharBuffer cbuf) { +// private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { +// byte[] barr = bbuf.array(); +// int bpos = bbuf.position() + bbuf.arrayOffset(); +// int blim = bbuf.limit() + bbuf.arrayOffset(); +// +// char[] carr = cbuf.array(); +// int cpos = cbuf.position() + cbuf.arrayOffset(); +// int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); +// while (bpos < blim && cpos < clim) { +// int b = barr[bpos++] & 0xFF; +// if (r > 0) { +// if ((b >> 6) != 0b10) +// throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); +// +// codePoint = (codePoint << 6) | (b & 0x3F); +// if (--r == 0) { +// if (codePoint > 0xFFFF) { // Surrogate pair +// int cp = codePoint - 0x10000; +// carr[cpos++] = (char) ((cp >> 10) + 0xD800); +// carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); +// } else +// carr[cpos++] = (char) codePoint; +// codePoint = 0; +// } +// } else if (b < 0x80) { // 1-byte ASCII +// carr[cpos++] = (char) barr[bpos++]; +// while (bpos < blim && (barr[bpos] & 0x80) == 0 && cpos < clim) +// carr[cpos++] = (char) barr[bpos++]; +// } else if (b < 0xc0) +// throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); +// else if (b < 0xe0) { +// r = 1; +// codePoint = b & 0x1F; +// } else if (b < 0xf0) { +// r = 2; +// codePoint = b & 0x0F; +// } else if (b <= 0xfa) { +// r = 3; +// codePoint = b & 0x07; +// } else +// throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); +// } +// bbuf.position(bpos - bbuf.arrayOffset()); +// cbuf.position(cpos - cbuf.arrayOffset()); +// } + + private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { + byte[] barr = bbuf.array(); + int bpos = bbuf.position() + bbuf.arrayOffset(); + int blim = bbuf.limit() + bbuf.arrayOffset(); + char[] carr = cbuf.array(); + int cpos = cbuf.position() + cbuf.arrayOffset(); + int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); + while (bpos < blim && cpos < clim) + cpos = fastAppend(barr[bpos++] & 0xFF, carr, cpos); + bbuf.position(bpos - bbuf.arrayOffset()); + cbuf.position(cpos - cbuf.arrayOffset()); + } + + private int fastAppend(int b, char[] cbuf, int cpos) { if (r > 0) { if ((b >> 6) != 0b10) throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); if (--r == 0) - append(cbuf); - return; - } - - if ((b >> 7) == 0) // 1-byte ASCII - cbuf.put((char) b); + return fastAppend(cbuf, cpos); + } else if ((b >> 7) == 0) // 1-byte ASCII + cbuf[cpos++] = (char) b; else if ((b >> 5) == 0b110) { r = 1; codePoint = b & 0x1F; @@ -51,9 +99,52 @@ else if ((b >> 5) == 0b110) { codePoint = b & 0x07; } else throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + return cpos; + } + + private int fastAppend(char[] cbuf, int cpos) { + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + cbuf[cpos++] = (char) ((cp >> 10) + 0xD800); + cbuf[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + cbuf[cpos++] = (char) codePoint; + codePoint = 0; + return cpos; + } + + private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { + while (bbuf.hasRemaining() && cbuf.remaining() > 1) { + slowAppend(bbuf.get() & 0xFF, cbuf); + } + } + + private void slowAppend(int b, CharBuffer cbuf) { + if (r > 0) { + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + + codePoint = (codePoint << 6) | (b & 0x3F); + if (--r == 0) + slowAppend(cbuf); + } else if (b < 0x80) // 1-byte ASCII + cbuf.put((char) b); + else if (b < 0xc0) + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + else if (b < 0xe0) { + r = 1; + codePoint = b & 0x1F; + } else if (b < 0xf0) { + r = 2; + codePoint = b & 0x0F; + } else if (b <= 0xfa) { + r = 3; + codePoint = b & 0x07; + } else + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); } - private void append(CharBuffer cbuf) { + private void slowAppend(CharBuffer cbuf) { if (codePoint > 0xFFFF) { // Surrogate pair int cp = codePoint - 0x10000; cbuf.put((char) ((cp >> 10) + 0xD800)).put((char) ((cp & 0x3FF) + 0xDC00)); From a245fc5dbb7a5ee4c97d9e7a9f2dbd4c6cb473ec Mon Sep 17 00:00:00 2001 From: Unknow Date: Fri, 27 Mar 2026 22:29:18 +0100 Subject: [PATCH 16/31] right fast decoder --- .../java/unknow/server/util/Utf8Decoder.java | 124 ++++++------------ 1 file changed, 41 insertions(+), 83 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index aacf7f48..9be6a8c9 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -20,99 +20,57 @@ public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { throw new IllegalArgumentException("Invalid UTF-8 string, truncated continuation bytes"); } -// private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { -// byte[] barr = bbuf.array(); -// int bpos = bbuf.position() + bbuf.arrayOffset(); -// int blim = bbuf.limit() + bbuf.arrayOffset(); -// -// char[] carr = cbuf.array(); -// int cpos = cbuf.position() + cbuf.arrayOffset(); -// int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); -// while (bpos < blim && cpos < clim) { -// int b = barr[bpos++] & 0xFF; -// if (r > 0) { -// if ((b >> 6) != 0b10) -// throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); -// -// codePoint = (codePoint << 6) | (b & 0x3F); -// if (--r == 0) { -// if (codePoint > 0xFFFF) { // Surrogate pair -// int cp = codePoint - 0x10000; -// carr[cpos++] = (char) ((cp >> 10) + 0xD800); -// carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); -// } else -// carr[cpos++] = (char) codePoint; -// codePoint = 0; -// } -// } else if (b < 0x80) { // 1-byte ASCII -// carr[cpos++] = (char) barr[bpos++]; -// while (bpos < blim && (barr[bpos] & 0x80) == 0 && cpos < clim) -// carr[cpos++] = (char) barr[bpos++]; -// } else if (b < 0xc0) -// throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); -// else if (b < 0xe0) { -// r = 1; -// codePoint = b & 0x1F; -// } else if (b < 0xf0) { -// r = 2; -// codePoint = b & 0x0F; -// } else if (b <= 0xfa) { -// r = 3; -// codePoint = b & 0x07; -// } else -// throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); -// } -// bbuf.position(bpos - bbuf.arrayOffset()); -// cbuf.position(cpos - cbuf.arrayOffset()); -// } - private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { byte[] barr = bbuf.array(); int bpos = bbuf.position() + bbuf.arrayOffset(); int blim = bbuf.limit() + bbuf.arrayOffset(); + char[] carr = cbuf.array(); int cpos = cbuf.position() + cbuf.arrayOffset(); int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); - while (bpos < blim && cpos < clim) - cpos = fastAppend(barr[bpos++] & 0xFF, carr, cpos); + while (bpos < blim && cpos < clim) { + int b = barr[bpos++] & 0xFF; + if (r > 0) { + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + + codePoint = (codePoint << 6) | (b & 0x3F); + if (--r == 0) { + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + carr[cpos++] = (char) ((cp >> 10) + 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + carr[cpos++] = (char) codePoint; + codePoint = 0; + } + } else if (b < 0x80) { // 1-byte ASCII + carr[cpos++] = (char) b; + while (bpos < blim && cpos < clim) { + b = barr[bpos] & 0xFF; + if (b >= 0x80) + break; + carr[cpos++] = (char) b; + bpos++; + } + } else if (b < 0xc0) + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + else if (b < 0xe0) { + r = 1; + codePoint = b & 0x1F; + } else if (b < 0xf0) { + r = 2; + codePoint = b & 0x0F; + } else if (b < 0xfb) { + r = 3; + codePoint = b & 0x07; + } else + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + } bbuf.position(bpos - bbuf.arrayOffset()); cbuf.position(cpos - cbuf.arrayOffset()); } - private int fastAppend(int b, char[] cbuf, int cpos) { - if (r > 0) { - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); - if (--r == 0) - return fastAppend(cbuf, cpos); - } else if ((b >> 7) == 0) // 1-byte ASCII - cbuf[cpos++] = (char) b; - else if ((b >> 5) == 0b110) { - r = 1; - codePoint = b & 0x1F; - } else if ((b >> 4) == 0b1110) { - r = 2; - codePoint = b & 0x0F; - } else if ((b >> 3) == 0b11110) { - r = 3; - codePoint = b & 0x07; - } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); - return cpos; - } - - private int fastAppend(char[] cbuf, int cpos) { - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - cbuf[cpos++] = (char) ((cp >> 10) + 0xD800); - cbuf[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - cbuf[cpos++] = (char) codePoint; - codePoint = 0; - return cpos; - } - private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { while (bbuf.hasRemaining() && cbuf.remaining() > 1) { slowAppend(bbuf.get() & 0xFF, cbuf); @@ -137,7 +95,7 @@ else if (b < 0xe0) { } else if (b < 0xf0) { r = 2; codePoint = b & 0x0F; - } else if (b <= 0xfa) { + } else if (b < 0xfb) { r = 3; codePoint = b & 0x07; } else From 480e595456dbc02a5ea7096f4299892f74c026bf Mon Sep 17 00:00:00 2001 From: Unknow Date: Fri, 27 Mar 2026 23:58:21 +0100 Subject: [PATCH 17/31] add bench on Encoder/Decoder --- .../unknow/server/bench/EncoderDecoder.java | 16 ++- .../server/servlet/utils/Utf8Encoder.java | 5 +- .../main/java/unknow/server/util/Decoder.java | 17 +++ .../unknow/server/util/DefaultDecoder.java | 21 --- .../main/java/unknow/server/util/Encoder.java | 40 ++++++ .../java/unknow/server/util/Utf8Decoder.java | 4 +- .../java/unknow/server/util/Utf8Encoder.java | 133 ++++++++++++++++++ 7 files changed, 209 insertions(+), 27 deletions(-) delete mode 100644 unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java create mode 100644 unknow-server-util/src/main/java/unknow/server/util/Encoder.java create mode 100644 unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index 5348970f..5a48aa89 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -10,17 +10,29 @@ import unknow.server.servlet.utils.Utf8Decoder; import unknow.server.servlet.utils.Utf8Encoder; +import unknow.server.util.Decoder; +import unknow.server.util.Encoder; public class EncoderDecoder { private static final String DATA = "Hello world!\nÇa va bien ?\nПривет, как дела?\n你好，世界\nこんにちは世界\n👋🌍✨🔥🚀\nLorem ipsum dolor sit amet, consectetur adipiscing elit."; @Benchmark - public String utf8Encoder() { + public String utf8EncoderServlet() { ByteBuffer b = ByteBuffer.allocate(4096); Utf8Encoder.encode(DATA, b); return new Utf8Decoder().append(b.array(), 0, b.position()).done(); } + @Benchmark + public String utf8Encoder() { + ByteBuffer b = ByteBuffer.allocate(4096); + Encoder.from(StandardCharsets.UTF_8).encode(CharBuffer.wrap(DATA), b, true); + + CharBuffer c = CharBuffer.allocate(4096); + Decoder.from(StandardCharsets.UTF_8).decode(b.flip(), c, true); + return c.flip().toString(); + } + @Benchmark public String charset() { ByteBuffer b = ByteBuffer.allocate(4096); @@ -29,7 +41,7 @@ public String charset() { b.flip(); CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); - CharBuffer c = CharBuffer.allocate(b.limit()); + CharBuffer c = CharBuffer.allocate(4096); d.decode(b, c, true); return c.flip().toString(); } diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/Utf8Encoder.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/Utf8Encoder.java index f21a37d6..6cb4b97c 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/Utf8Encoder.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/utils/Utf8Encoder.java @@ -46,7 +46,8 @@ public byte next() { code = ((code << 10) + str.charAt(i++)) + (0x10000 - (0xD800 << 10) - 0xDC00); else code = 0xFFFD; - } + } else if (code >= 0xDC00 && code <= 0xDFFF) + code = 0xFFFD; count++; if (code <= 0x7F) @@ -56,7 +57,7 @@ else if (code <= 0x7FF) { return (byte) (0xC0 | (code >> 6)); } if (code <= 0xFFFF) { - tmp[r++] = (byte) (0x80 | (0x80 | (code & 0x3F))); + tmp[r++] = (byte) (0x80 | (code & 0x3F)); tmp[r++] = (byte) (0x80 | ((code >> 6) & 0x3F)); return (byte) (0xE0 | (code >> 12)); } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Decoder.java index 6c001036..f8d90b02 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Decoder.java @@ -3,6 +3,8 @@ import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; import java.nio.charset.StandardCharsets; public interface Decoder { @@ -20,4 +22,19 @@ public static Decoder from(Charset charset) { * @param endOfInput the data is done */ void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput); + + public class DefaultDecoder implements Decoder { + private final CharsetDecoder dec; + + public DefaultDecoder(CharsetDecoder dec) { + this.dec = dec; + } + + @Override + public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { + CoderResult r = dec.decode(bbuf, cbuf, endOfInput); + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } + } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java b/unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java deleted file mode 100644 index dde42bba..00000000 --- a/unknow-server-util/src/main/java/unknow/server/util/DefaultDecoder.java +++ /dev/null @@ -1,21 +0,0 @@ -package unknow.server.util; - -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; - -public class DefaultDecoder implements Decoder { - private final CharsetDecoder dec; - - public DefaultDecoder(CharsetDecoder dec) { - this.dec = dec; - } - - @Override - public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { - CoderResult r = dec.decode(bbuf, cbuf, endOfInput); - if (r.isError()) - throw new IllegalArgumentException(r.toString()); - } -} diff --git a/unknow-server-util/src/main/java/unknow/server/util/Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Encoder.java new file mode 100644 index 00000000..5c837dc6 --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/Encoder.java @@ -0,0 +1,40 @@ +package unknow.server.util; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.StandardCharsets; + +public interface Encoder { + + public static Encoder from(Charset charset) { + if (charset.equals(StandardCharsets.UTF_8)) + return new Utf8Encoder(); + return new DefaultEncoder(charset.newEncoder()); + } + + /** + * append one byte + * @param cbuf char to encode + * @param bbuf byte output + * @param endOfInput the data is done + */ + void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput); + + public class DefaultEncoder implements Encoder { + private final CharsetEncoder enc; + + public DefaultEncoder(CharsetEncoder dec) { + this.enc = dec; + } + + @Override + public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { + CoderResult r = enc.encode(cbuf, bbuf, endOfInput); + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } + } +} diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 9be6a8c9..255dedbf 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -47,8 +47,8 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { } else if (b < 0x80) { // 1-byte ASCII carr[cpos++] = (char) b; while (bpos < blim && cpos < clim) { - b = barr[bpos] & 0xFF; - if (b >= 0x80) + b = barr[bpos]; + if (b < 0) break; carr[cpos++] = (char) b; bpos++; diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java new file mode 100644 index 00000000..de21e59f --- /dev/null +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -0,0 +1,133 @@ +package unknow.server.util; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; + +public class Utf8Encoder implements Encoder { + private int surrogate; + + @Override + public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { + if (!cbuf.hasRemaining() || bbuf.remaining() < 4) + return; + if (cbuf.hasArray() && bbuf.hasArray()) + fastEncode(cbuf, bbuf, endOfInput); + else + slowEncode(cbuf, bbuf, endOfInput); + } + + private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { + char[] carr = cbuf.array(); + int cpos = cbuf.position() + cbuf.arrayOffset(); + int clim = cbuf.limit() + cbuf.arrayOffset(); + + byte[] barr = bbuf.array(); + int bpos = bbuf.position() + bbuf.arrayOffset(); + int blim = bbuf.limit() - 4 + bbuf.arrayOffset(); + + if (surrogate != 0) { + int low = carr[cpos++]; + int code; + if (low < 0xDC00 || low > 0xDFFF) + code = 0xFFFD; + else + code = 0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00); + if (code <= 0x7F) + barr[bpos++] = (byte) code; + else if (code <= 0x7FF) { + barr[bpos++] = (byte) (0xC0 | (code >> 6)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } else if (code <= 0xFFFF) { + barr[bpos++] = (byte) (0xE0 | (code >> 12)); + barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } else { + barr[bpos++] = (byte) (0xF0 | (code >> 18)); + barr[bpos++] = (byte) (0x80 | ((code >> 12) & 0x3F)); + barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } + surrogate = 0; + } + + loop: while (cpos < clim && bpos < blim) { + int code = carr[cpos++]; + while (code <= 0x7F) { + barr[bpos++] = (byte) code; + if (cpos == clim || bpos == blim) + break loop; + code = carr[cpos++]; + } + if (code >= 0xD800 && code <= 0xDBFF) { + if (cpos < clim) + code = 0x10000 + ((code - 0xD800) << 10) + (carr[cpos++] - 0xDC00); + else if (endOfInput) + code = 0xFFFD; + else { + surrogate = code; + return; + } + } else if (code >= 0xDC00 && code <= 0xDFFF) + code = 0xFFFD; + if (code <= 0x7FF) { + barr[bpos++] = (byte) (0xC0 | (code >> 6)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } else if (code <= 0xFFFF) { + barr[bpos++] = (byte) (0xE0 | (code >> 12)); + barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } else { + barr[bpos++] = (byte) (0xF0 | (code >> 18)); + barr[bpos++] = (byte) (0x80 | ((code >> 12) & 0x3F)); + barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } + } + cbuf.position(cpos - cbuf.arrayOffset()); + bbuf.position(bpos - bbuf.arrayOffset()); + } + + private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { + if (surrogate != 0) { + int low = cbuf.get(); + if (low < 0xDC00 || low > 0xDFFF) + slowAppend(0xFFFD, bbuf); + else + slowAppend(0x10000 + ((surrogate - 0xD800) << 10) + (cbuf.get() - 0xDC00), bbuf); + surrogate = 0; + } + while (cbuf.hasRemaining() && bbuf.remaining() >= 4) { + int code = cbuf.get(); + if (code >= 0xD800 && code <= 0xDBFF) { + if (cbuf.hasRemaining()) + code = 0x10000 + ((code - 0xD800) << 10) + (cbuf.get() - 0xDC00); + else if (endOfInput) + code = 0xFFFD; + else { + surrogate = code; + return; + } + } else if (code >= 0xDC00 && code <= 0xDFFF) + code = 0xFFFD; + slowAppend(code, bbuf); + } + } + + private void slowAppend(int code, ByteBuffer bbuf) { + if (code <= 0x7F) + bbuf.put((byte) code); + else if (code <= 0x7FF) { + bbuf.put((byte) (0xC0 | (code >> 6))); + bbuf.put((byte) (0x80 | (code & 0x3F))); + } else if (code <= 0xFFFF) { + bbuf.put((byte) (0xE0 | (code >> 12))); + bbuf.put((byte) (0x80 | ((code >> 6) & 0x3F))); + bbuf.put((byte) (0x80 | (code & 0x3F))); + } else { + bbuf.put((byte) (0xF0 | (code >> 18))); + bbuf.put((byte) (0x80 | ((code >> 12) & 0x3F))); + bbuf.put((byte) (0x80 | ((code >> 6) & 0x3F))); + bbuf.put((byte) (0x80 | (code & 0x3F))); + } + } +} From e4034576ea71095835af0f005229395a8dfec0d5 Mon Sep 17 00:00:00 2001 From: Unknow Date: Mon, 30 Mar 2026 00:12:28 +0200 Subject: [PATCH 18/31] update utf8 dec/enc --- .../unknow/server/bench/EncoderDecoder.java | 214 ++++++++++++++++-- .../main/java/unknow/server/bench/Main.java | 10 +- .../server/servlet/impl/ServletWriter.java | 20 +- .../java/unknow/server/util/Utf8Decoder.java | 163 ++++++++++--- .../java/unknow/server/util/Utf8Encoder.java | 99 ++++---- 5 files changed, 398 insertions(+), 108 deletions(-) diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index 5a48aa89..655c9ecf 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -7,48 +7,220 @@ import java.nio.charset.StandardCharsets; import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; -import unknow.server.servlet.utils.Utf8Decoder; -import unknow.server.servlet.utils.Utf8Encoder; import unknow.server.util.Decoder; import unknow.server.util.Encoder; public class EncoderDecoder { - private static final String DATA = "Hello world!\nÇa va bien ?\nПривет, как дела?\n你好，世界\nこんにちは世界\n👋🌍✨🔥🚀\nLorem ipsum dolor sit amet, consectetur adipiscing elit."; + private static final String LATIN = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet." + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed non risus. Suspendisse lectus tortor, dignissim sit amet, adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam. Maecenas ligula massa, varius a, semper congue, euismod non, mi. Proin porttitor, orci nec nonummy molestie, enim est eleifend mi, non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa, scelerisque vitae, consequat in, pretium a, enim. Pellentesque congue. Ut in risus volutpat libero pharetra tempor. Cras vestibulum bibendum augue. Praesent egestas leo in pede. Praesent blandit odio eu enim. Pellentesque sed dui ut augue blandit sodales. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam nibh. Mauris ac mauris sed pede pellentesque fermentum. Maecenas adipiscing ante non diam sodales hendrerit.\r\n" + + "\r\n" + + "Ut velit mauris, egestas sed, gravida nec, ornare ut, mi. Aenean ut orci vel massa suscipit pulvinar. Nulla sollicitudin. Fusce varius, ligula non tempus aliquam, nunc turpis ullamcorper nibh, in tempus sapien eros vitae ligula. Pellentesque rhoncus nunc et augue. Integer id felis. Curabitur aliquet pellentesque diam. Integer quis metus vitae elit lobortis egestas. Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Morbi vel erat non mauris convallis vehicula. Nulla et sapien. Integer tortor tellus, aliquam faucibus, convallis id, congue eu, quam. Mauris ullamcorper felis vitae erat. Proin feugiat, augue non elementum posuere, metus purus iaculis lectus, et tristique ligula justo vitae magna.\r\n" + + "Aliquam convallis sollicitudin purus. Praesent aliquam, enim at fermentum mollis, ligula massa adipiscing nisl, ac euismod nibh nisl eu lectus. Fusce vulputate sem at sapien. Vivamus leo. Aliquam euismod libero eu enim. Nulla nec felis sed leo placerat imperdiet. Aenean suscipit nulla in justo. Suspendisse cursus rutrum augue. Nulla tincidunt tincidunt mi. Curabitur iaculis, lorem vel rhoncus faucibus, felis magna fermentum augue, et ultricies lacus lorem varius purus. Curabitur eu amet."; + + private static final String SIMPLE = "Hello world!\nÇa va bien ?\nПривет, как дела?\n你好，世界\nこんにちは世界\n👋🌍✨🔥🚀\nLorem ipsum dolor sit amet, consectetur adipiscing elit."; + + private static final String COMPLEX = "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱"; + + @State(Scope.Thread) + public static class Data { + @Param({ /* "latin", "simple",*/ "complex" }) + String name; + CharBuffer cbuf; + ByteBuffer bytes; + + @Setup + public void setup() { + switch (name) { + case "latin": + init(LATIN); + break; + case "simple": + init(SIMPLE); + break; + case "complex": + init(COMPLEX); + break; + } + } + + private void init(String str) { + cbuf = CharBuffer.allocate(str.length()); + str.getChars(0, str.length(), cbuf.array(), 0); + bytes = ByteBuffer.wrap(str.getBytes(StandardCharsets.UTF_8)); + } + } + + public static void main(String[] arg) throws Throwable { + Data data = new Data(); + data.init(COMPLEX); + new EncoderDecoder().decoderUtf8(data); + } + +// @Benchmark +// public ByteBuffer encoderUtf8Servlet(EncodeData data) { +// ByteBuffer b = ByteBuffer.allocate(4096); +// Utf8Encoder.encode(data.str, b); +// return b.flip(); +// } +// +// @Benchmark +// public String decoderUtf8Servlet(DecodeData data) { +// ByteBuffer b = ByteBuffer.wrap(data.bytes); +// return new Utf8Decoder().append(b.array(), 0, b.limit()).done(); +// } @Benchmark - public String utf8EncoderServlet() { + public ByteBuffer encoderUtf8(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); - Utf8Encoder.encode(DATA, b); - return new Utf8Decoder().append(b.array(), 0, b.position()).done(); + Encoder e = Encoder.from(StandardCharsets.UTF_8); + CharBuffer cbuf = data.cbuf.duplicate(); + while (cbuf.hasRemaining()) { + e.encode(cbuf, b, false); + b.clear(); + } + return b.clear(); + } + + @Benchmark + public CharBuffer decoderUtf8(Data data) { + CharBuffer c = CharBuffer.allocate(4096); + Decoder d = Decoder.from(StandardCharsets.UTF_8); + ByteBuffer bbuf = data.bytes.duplicate(); + while (bbuf.hasRemaining()) { + d.decode(bbuf, c, false); + c.clear(); + } + return c.clear(); } @Benchmark - public String utf8Encoder() { + public ByteBuffer encoderUtf8Slow(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); - Encoder.from(StandardCharsets.UTF_8).encode(CharBuffer.wrap(DATA), b, true); + Encoder e = Encoder.from(StandardCharsets.UTF_8); + CharBuffer cbuf = data.cbuf.asReadOnlyBuffer(); + while (cbuf.hasRemaining()) { + e.encode(cbuf, b, false); + b.clear(); + } + return b.clear(); + } + @Benchmark + public CharBuffer decoderUtf8Slow(Data data) { CharBuffer c = CharBuffer.allocate(4096); - Decoder.from(StandardCharsets.UTF_8).decode(b.flip(), c, true); - return c.flip().toString(); + Decoder d = Decoder.from(StandardCharsets.UTF_8); + ByteBuffer bbuf = data.bytes.asReadOnlyBuffer(); + while (bbuf.hasRemaining()) { + d.decode(bbuf, c, false); + c.clear(); + } + return c.clear(); } @Benchmark - public String charset() { + public ByteBuffer encoderCharset(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); CharsetEncoder e = StandardCharsets.UTF_8.newEncoder(); - e.encode(CharBuffer.wrap(DATA), b, true); - - b.flip(); - CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); - CharBuffer c = CharBuffer.allocate(4096); - d.decode(b, c, true); - return c.flip().toString(); + CharBuffer cbuf = data.cbuf.duplicate(); + while (cbuf.hasRemaining()) { + e.encode(cbuf, b, false); + b.clear(); + } + return b.clear(); } @Benchmark - public String string() { - byte[] bytes = DATA.getBytes(StandardCharsets.UTF_8); - return new String(bytes, StandardCharsets.UTF_8); + public CharBuffer decoderCharset(Data data) { + CharBuffer c = CharBuffer.allocate(4096); + CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); + ByteBuffer bbuf = data.bytes.duplicate(); + while (bbuf.hasRemaining()) { + d.decode(bbuf, c, false); + c.clear(); + } + return c.clear(); } } diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/Main.java b/unknow-server-bench/src/main/java/unknow/server/bench/Main.java index 8029c2f5..31de0ea6 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/Main.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/Main.java @@ -17,13 +17,15 @@ public class Main { public static void main(String[] args) throws Exception { - Options o = new OptionsBuilder().forks(1).measurementIterations(10).verbosity(VerboseMode.SILENT).warmupIterations(5).build(); - try (PrintStream w = args.length > 0 ? new PrintStream(Files.newOutputStream(Paths.get(args[0])), false, StandardCharsets.UTF_8) : System.out) { - for (Class c : Arrays.asList(EncoderDecoder.class, BenchJaxb.class, BenchDocument.class, BenchProtostuff.class)) { + Options o = new OptionsBuilder().forks(1).measurementIterations(10).verbosity(VerboseMode.NORMAL).warmupIterations(5).build(); + + try (PrintStream w = args.length > 0 ? new PrintStream(Files.newOutputStream(Paths.get(args[0])), false, StandardCharsets.UTF_8) : System.err) { + for (Class c : Arrays.asList(EncoderDecoder.class/*, BenchJaxb.class, BenchDocument.class, BenchProtostuff.class*/)) { + Collection result = new Runner(new OptionsBuilder().parent(o).include(c.getName()).build()).run(); + w.println(c.getSimpleName()); w.println("```"); - Collection result = new Runner(new OptionsBuilder().parent(o).include(c.getName()).build()).run(); ResultFormatFactory.getInstance(ResultFormatType.TEXT, w).writeOut(result); w.println("```"); } diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java index 1dd83e37..287b1d4d 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java @@ -4,30 +4,26 @@ import java.io.Writer; import java.nio.CharBuffer; import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CoderResult; -import java.nio.charset.CodingErrorAction; + +import unknow.server.util.Encoder; public class ServletWriter extends Writer { private final AbstractServletOutput out; - private final CharsetEncoder enc; + private final Encoder enc; public ServletWriter(AbstractServletOutput out, Charset c) { this.out = out; - this.enc = c.newEncoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT); + this.enc = Encoder.from(c); } public void write(CharBuffer cbuf) throws IOException { if (out.isClosed()) throw new IOException("closed"); - CoderResult r = enc.encode(cbuf, out.buffer, false); - if (r.isError()) - throw new IOException("Invalid caracter found"); - while (r.isOverflow()) { + + enc.encode(cbuf, out.buffer, false); + while (cbuf.hasRemaining()) { out.flush(); - r = enc.encode(cbuf, out.buffer, false); - if (r.isError()) - throw new IOException("Invalid caracter found"); + enc.encode(cbuf, out.buffer, false); } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 255dedbf..e9138833 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -6,6 +6,7 @@ public class Utf8Decoder implements Decoder { /** code point in building */ private int codePoint; + /** remaining octet to read */ private int r; @@ -28,12 +29,140 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { char[] carr = cbuf.array(); int cpos = cbuf.position() + cbuf.arrayOffset(); int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); - while (bpos < blim && cpos < clim) { + + int maxAscii = Math.min(clim, cpos + blim - bpos); + int codePoint = this.codePoint; + int r = this.r; + + if (r > 0) { + if (bpos + r <= blim) { // can read all + int b; + switch (r) { + case 3: + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + case 2: + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + case 1: + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + } + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + carr[cpos++] = (char) ((cp >> 10) + 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + carr[cpos++] = (char) codePoint; + r = 0; + } else { + while (bpos < blim && cpos < clim) { + int b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + if (--r == 0) { + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + carr[cpos++] = (char) ((cp >> 10) + 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + carr[cpos++] = (char) codePoint; + } + } + } + } + + while (bpos < blim && cpos < clim) + + { int b = barr[bpos++] & 0xFF; - if (r > 0) { + if (b < 0x80) { // 1-byte ASCII + carr[cpos++] = (char) b; + while (cpos < maxAscii && barr[bpos] >= 0) + carr[cpos++] = (char) barr[bpos++]; + } else if (b < 0xc0) + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + else { + if (b < 0xe0) { + codePoint = b & 0x1F; + if (bpos < blim) { + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + carr[cpos++] = (char) ((cp >> 10) + 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + carr[cpos++] = (char) codePoint; + } else { + r = 1; + break; + } + } else if (b < 0xf0) { + codePoint = b & 0x0F; + if (bpos + 1 < blim) { + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + carr[cpos++] = (char) ((cp >> 10) + 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + carr[cpos++] = (char) codePoint; + } else { + r = 2; + break; + } + } else if (b < 0xf4) { + codePoint = b & 0x07; + if (bpos + 2 < blim) { + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + b = barr[bpos++] & 0xFF; + if ((b >> 6) != 0b10) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint > 0xFFFF) { // Surrogate pair + int cp = codePoint - 0x10000; + carr[cpos++] = (char) ((cp >> 10) + 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); + } else + carr[cpos++] = (char) codePoint; + } else { + r = 3; + break; + } + } else + throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + } + } + + if (r > 0) { + while (bpos < blim && cpos < clim) { + int b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); if (--r == 0) { if (codePoint > 0xFFFF) { // Surrogate pair @@ -42,31 +171,13 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); } else carr[cpos++] = (char) codePoint; - codePoint = 0; - } - } else if (b < 0x80) { // 1-byte ASCII - carr[cpos++] = (char) b; - while (bpos < blim && cpos < clim) { - b = barr[bpos]; - if (b < 0) - break; - carr[cpos++] = (char) b; - bpos++; } - } else if (b < 0xc0) - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); - else if (b < 0xe0) { - r = 1; - codePoint = b & 0x1F; - } else if (b < 0xf0) { - r = 2; - codePoint = b & 0x0F; - } else if (b < 0xfb) { - r = 3; - codePoint = b & 0x07; - } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + } } + + this.r = r; + this.codePoint = codePoint; + bbuf.position(bpos - bbuf.arrayOffset()); cbuf.position(cpos - cbuf.arrayOffset()); } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index de21e59f..2b2a28c3 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -23,64 +23,68 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { byte[] barr = bbuf.array(); int bpos = bbuf.position() + bbuf.arrayOffset(); - int blim = bbuf.limit() - 4 + bbuf.arrayOffset(); + int blim = bbuf.limit() - 3 + bbuf.arrayOffset(); if (surrogate != 0) { int low = carr[cpos++]; - int code; - if (low < 0xDC00 || low > 0xDFFF) - code = 0xFFFD; - else - code = 0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00); - if (code <= 0x7F) - barr[bpos++] = (byte) code; - else if (code <= 0x7FF) { - barr[bpos++] = (byte) (0xC0 | (code >> 6)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); - } else if (code <= 0xFFFF) { - barr[bpos++] = (byte) (0xE0 | (code >> 12)); - barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); - } else { + if (low >= 0xDC00 && low <= 0xDFFF) { + int code = 0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00); barr[bpos++] = (byte) (0xF0 | (code >> 18)); barr[bpos++] = (byte) (0x80 | ((code >> 12) & 0x3F)); barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } else { + barr[bpos++] = (byte) 0xEF; + barr[bpos++] = (byte) 0xBF; + barr[bpos++] = (byte) 0xbd; } surrogate = 0; } - - loop: while (cpos < clim && bpos < blim) { + int maxAscii = Math.min(clim, cpos + blim - bpos); + while (cpos < maxAscii && carr[cpos] <= 0x7f) + barr[bpos++] = (byte) carr[cpos++]; + while (cpos < clim && bpos < blim) { int code = carr[cpos++]; - while (code <= 0x7F) { + if (code <= 0x7f) { barr[bpos++] = (byte) code; - if (cpos == clim || bpos == blim) - break loop; - code = carr[cpos++]; - } - if (code >= 0xD800 && code <= 0xDBFF) { - if (cpos < clim) - code = 0x10000 + ((code - 0xD800) << 10) + (carr[cpos++] - 0xDC00); - else if (endOfInput) - code = 0xFFFD; - else { - surrogate = code; - return; - } - } else if (code >= 0xDC00 && code <= 0xDFFF) - code = 0xFFFD; - if (code <= 0x7FF) { + while (cpos < maxAscii && carr[cpos] <= 0x7f) + barr[bpos++] = (byte) carr[cpos++]; + } else if (code <= 0x7FF) { barr[bpos++] = (byte) (0xC0 | (code >> 6)); barr[bpos++] = (byte) (0x80 | (code & 0x3F)); - } else if (code <= 0xFFFF) { - barr[bpos++] = (byte) (0xE0 | (code >> 12)); - barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); } else { - barr[bpos++] = (byte) (0xF0 | (code >> 18)); - barr[bpos++] = (byte) (0x80 | ((code >> 12) & 0x3F)); - barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + int i = code >> 10; + if (i == 0b110110) { // high surrogate + if (cpos < clim) { + int low = carr[cpos++]; + if (low >= 0xDC00 && low <= 0xDFFF) { + int c = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); + barr[bpos++] = (byte) (0xF0 | (c >> 18)); + barr[bpos++] = (byte) (0x80 | ((c >> 12) & 0x3F)); + barr[bpos++] = (byte) (0x80 | ((c >> 6) & 0x3F)); + barr[bpos++] = (byte) (0x80 | (c & 0x3F)); + } else { + barr[bpos++] = (byte) 0xEF; + barr[bpos++] = (byte) 0xBF; + barr[bpos++] = (byte) 0xbd; + } + } else if (endOfInput) { + barr[bpos++] = (byte) 0xEF; + barr[bpos++] = (byte) 0xBF; + barr[bpos++] = (byte) 0xbd; + } else { + surrogate = code; + break; + } + } else if (i == 0b110111) { // lone low surrogate + barr[bpos++] = (byte) 0xEF; + barr[bpos++] = (byte) 0xBF; + barr[bpos++] = (byte) 0xbd; + } else { + barr[bpos++] = (byte) (0xE0 | (code >> 12)); + barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); + barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + } } } cbuf.position(cpos - cbuf.arrayOffset()); @@ -96,9 +100,14 @@ private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { slowAppend(0x10000 + ((surrogate - 0xD800) << 10) + (cbuf.get() - 0xDC00), bbuf); surrogate = 0; } + int cpos = bbuf.position(); + int maxAscii = cpos + Math.min(cbuf.remaining(), bbuf.remaining()); + while (cpos < maxAscii && cbuf.get(cpos++) <= 0x7f) + bbuf.put((byte) cbuf.get()); while (cbuf.hasRemaining() && bbuf.remaining() >= 4) { int code = cbuf.get(); - if (code >= 0xD800 && code <= 0xDBFF) { + int i = code >> 10; + if (i == 0b110110) { // high surrogate if (cbuf.hasRemaining()) code = 0x10000 + ((code - 0xD800) << 10) + (cbuf.get() - 0xDC00); else if (endOfInput) @@ -107,7 +116,7 @@ else if (endOfInput) surrogate = code; return; } - } else if (code >= 0xDC00 && code <= 0xDFFF) + } else if (i == 0b110111) // lone low surrogate code = 0xFFFD; slowAppend(code, bbuf); } From d3ca1bd20f0219a21c070ea2ff6e86c7c5628a60 Mon Sep 17 00:00:00 2001 From: Unknow Date: Mon, 30 Mar 2026 21:46:38 +0200 Subject: [PATCH 19/31] add some fix and --- .../unknow/server/bench/EncoderDecoder.java | 2 +- .../server/servlet/impl/ServletWriter.java | 4 +- .../java/unknow/server/util/Utf8Decoder.java | 45 ++++-- .../java/unknow/server/util/Utf8Encoder.java | 28 ++-- .../unknow/server/util/Utf8DecoderTest.java | 131 ++++++++++++++++++ .../unknow/server/util/Utf8EncoderTest.java | 75 ++++++++++ 6 files changed, 258 insertions(+), 27 deletions(-) create mode 100644 unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java create mode 100644 unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index 655c9ecf..d14b8e80 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -106,7 +106,7 @@ public class EncoderDecoder { @State(Scope.Thread) public static class Data { - @Param({ /* "latin", "simple",*/ "complex" }) + @Param({ "latin", "simple", "complex" }) String name; CharBuffer cbuf; ByteBuffer bytes; diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java index 287b1d4d..7010f788 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java @@ -9,10 +9,12 @@ public class ServletWriter extends Writer { private final AbstractServletOutput out; + private final Charset c; private final Encoder enc; public ServletWriter(AbstractServletOutput out, Charset c) { this.out = out; + this.c = c; this.enc = Encoder.from(c); } @@ -40,7 +42,7 @@ public void write(int c) throws IOException { public void write(String str, int off, int len) throws IOException { if (out.isClosed()) throw new IOException("closed"); - write(CharBuffer.wrap(str, off, off + len)); + out.write(str.getBytes(c)); } @Override diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index e9138833..d6f84859 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -43,11 +43,15 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { if ((b >> 6) != 0b10) throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint < 0x10) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); case 2: b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint < 0x20) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); case 1: b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) @@ -67,6 +71,8 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { if ((b >> 6) != 0b10) throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); codePoint = (codePoint << 6) | (b & 0x3F); + if ((r == 2 && codePoint < 0x20) || (r == 3 && codePoint < 0x10)) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); if (--r == 0) { if (codePoint > 0xFFFF) { // Surrogate pair int cp = codePoint - 0x10000; @@ -79,23 +85,23 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { } } - while (bpos < blim && cpos < clim) - - { + while (bpos < blim && cpos < clim) { int b = barr[bpos++] & 0xFF; if (b < 0x80) { // 1-byte ASCII carr[cpos++] = (char) b; while (cpos < maxAscii && barr[bpos] >= 0) carr[cpos++] = (char) barr[bpos++]; } else if (b < 0xc0) - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); else { if (b < 0xe0) { codePoint = b & 0x1F; + if (codePoint < 2) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); if (bpos < blim) { b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); if (codePoint > 0xFFFF) { // Surrogate pair int cp = codePoint - 0x10000; @@ -112,11 +118,13 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { if (bpos + 1 < blim) { b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint < 0x20) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); if (codePoint > 0xFFFF) { // Surrogate pair int cp = codePoint - 0x10000; @@ -133,15 +141,18 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { if (bpos + 2 < blim) { b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); + if (codePoint < 0x10) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); if (codePoint > 0xFFFF) { // Surrogate pair int cp = codePoint - 0x10000; @@ -154,7 +165,7 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { break; } } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } } @@ -162,7 +173,7 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { while (bpos < blim && cpos < clim) { int b = barr[bpos++] & 0xFF; if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); if (--r == 0) { if (codePoint > 0xFFFF) { // Surrogate pair @@ -191,18 +202,22 @@ private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { private void slowAppend(int b, CharBuffer cbuf) { if (r > 0) { if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); codePoint = (codePoint << 6) | (b & 0x3F); + if ((r == 2 && codePoint < 0x20) || (r == 3 && codePoint < 0x10)) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); if (--r == 0) slowAppend(cbuf); } else if (b < 0x80) // 1-byte ASCII cbuf.put((char) b); else if (b < 0xc0) - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); else if (b < 0xe0) { r = 1; codePoint = b & 0x1F; + if (codePoint < 2) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); } else if (b < 0xf0) { r = 2; codePoint = b & 0x0F; @@ -210,7 +225,7 @@ else if (b < 0xe0) { r = 3; codePoint = b & 0x07; } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: " + b); + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } private void slowAppend(CharBuffer cbuf) { diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index 2b2a28c3..814081be 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -26,8 +26,9 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { int blim = bbuf.limit() - 3 + bbuf.arrayOffset(); if (surrogate != 0) { - int low = carr[cpos++]; + int low = carr[cpos]; if (low >= 0xDC00 && low <= 0xDFFF) { + cpos++; int code = 0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00); barr[bpos++] = (byte) (0xF0 | (code >> 18)); barr[bpos++] = (byte) (0x80 | ((code >> 12) & 0x3F)); @@ -56,8 +57,9 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { int i = code >> 10; if (i == 0b110110) { // high surrogate if (cpos < clim) { - int low = carr[cpos++]; + int low = carr[cpos]; if (low >= 0xDC00 && low <= 0xDFFF) { + cpos++; int c = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); barr[bpos++] = (byte) (0xF0 | (c >> 18)); barr[bpos++] = (byte) (0x80 | ((c >> 12) & 0x3F)); @@ -93,24 +95,30 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { if (surrogate != 0) { - int low = cbuf.get(); - if (low < 0xDC00 || low > 0xDFFF) + int low = cbuf.get(cbuf.position()); + if (low >= 0xDC00 && low <= 0xDFFF) { + slowAppend(0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00), bbuf); + cbuf.position(cbuf.position() + 1); + } else slowAppend(0xFFFD, bbuf); - else - slowAppend(0x10000 + ((surrogate - 0xD800) << 10) + (cbuf.get() - 0xDC00), bbuf); surrogate = 0; } int cpos = bbuf.position(); - int maxAscii = cpos + Math.min(cbuf.remaining(), bbuf.remaining()); + int maxAscii = Math.min(cbuf.limit(), cpos + bbuf.remaining()); while (cpos < maxAscii && cbuf.get(cpos++) <= 0x7f) bbuf.put((byte) cbuf.get()); while (cbuf.hasRemaining() && bbuf.remaining() >= 4) { int code = cbuf.get(); int i = code >> 10; if (i == 0b110110) { // high surrogate - if (cbuf.hasRemaining()) - code = 0x10000 + ((code - 0xD800) << 10) + (cbuf.get() - 0xDC00); - else if (endOfInput) + if (cbuf.hasRemaining()) { + int low = cbuf.get(cbuf.position()); + if (low >= 0xDC00 && low <= 0xDFFF) { + code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); + cbuf.position(cbuf.position() + 1); + } else + code = 0xFFFD; + } else if (endOfInput) code = 0xFFFD; else { surrogate = code; diff --git a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java new file mode 100644 index 00000000..442828a1 --- /dev/null +++ b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java @@ -0,0 +1,131 @@ +package unknow.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +public class Utf8DecoderTest { + static Stream utf8Strings() { + return Stream.of("", "hello", "éàç", "こんにちは", "😀😃😄😁", "hello é 😀", "𐍈"); + } + + static Stream invalidUtf8() { + return Stream.of(new byte[] { (byte) 0x80 }, new byte[] { (byte) 0xC0 }, new byte[] { (byte) 0xE0, (byte) 0x80 }, + //new byte[] { (byte) 0xF0, (byte) 0x80, (byte) 0x80 }, + new byte[] { (byte) 0xC0, (byte) 0xAF }, new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, new byte[] { (byte) 0xF8 }, new byte[] { (byte) 0xFF }); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPath(String input) { + Decoder decoder = new Utf8Decoder(); + ByteBuffer bbuf = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)); + CharBuffer cbuf = CharBuffer.allocate(100); + + decoder.decode(bbuf, cbuf, true); + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPathChuncked(String input) { + Decoder decoder = new Utf8Decoder(); + + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + CharBuffer cbuf = CharBuffer.allocate(100); + + for (int i = 0; i < bytes.length; i++) { + ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { bytes[i] }); + decoder.decode(bbuf, cbuf, i == bytes.length - 1); + } + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testFastPathError(byte[] input) { + Decoder decoder = new Utf8Decoder(); + + ByteBuffer bbuf = ByteBuffer.wrap(input); + CharBuffer cbuf = CharBuffer.allocate(10); + + assertThrows(IllegalArgumentException.class, () -> decoder.decode(bbuf, cbuf, true)); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testFastPathErrorChuncked(byte[] input) { + Decoder decoder = Decoder.from(StandardCharsets.UTF_8); + + CharBuffer cbuf = CharBuffer.allocate(10); + + assertThrows(IllegalArgumentException.class, () -> { + for (int i = 0; i < input.length; i++) { + ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { input[i] }); + decoder.decode(bbuf, cbuf, i == input.length - 1); + } + }); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPath(String input) { + ByteBuffer bbuf = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)).asReadOnlyBuffer(); + CharBuffer cbuf = CharBuffer.allocate(100); + + new Utf8Decoder().decode(bbuf, cbuf, true); + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPathChuncked(String input) { + Decoder decoder = new Utf8Decoder(); + + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + CharBuffer cbuf = CharBuffer.allocate(100); + + for (int i = 0; i < bytes.length; i++) { + ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { bytes[i] }).asReadOnlyBuffer(); + decoder.decode(bbuf, cbuf, i == bytes.length - 1); + } + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testSlowPathError(byte[] input) { + Decoder decoder = new Utf8Decoder(); + + ByteBuffer bbuf = ByteBuffer.wrap(input).asReadOnlyBuffer(); + CharBuffer cbuf = CharBuffer.allocate(10); + + assertThrows(IllegalArgumentException.class, () -> decoder.decode(bbuf, cbuf, true)); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testSlowPathErrorChuncked(byte[] input) { + Decoder decoder = Decoder.from(StandardCharsets.UTF_8); + + CharBuffer cbuf = CharBuffer.allocate(10); + + assertThrows(IllegalArgumentException.class, () -> { + for (int i = 0; i < input.length; i++) { + ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { input[i] }).asReadOnlyBuffer(); + decoder.decode(bbuf, cbuf, i == input.length - 1); + } + }); + } +} diff --git a/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java new file mode 100644 index 00000000..2be8e468 --- /dev/null +++ b/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java @@ -0,0 +1,75 @@ +package unknow.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class Utf8EncoderTest { + static Stream utf8Strings() { + return Stream.of(Arguments.of("", ""), Arguments.of("hello", "hello"), Arguments.of("éàç", "éàç"), Arguments.of("こんにちは", "こんにちは"), + Arguments.of("😀😃😄😁", "😀😃😄😁"), Arguments.of("hello é 😀", "hello é 😀"), Arguments.of("𐍈", "𐍈"), Arguments.of("\uD800", "\uFFFD"), + Arguments.of("\uDC00", "\uFFFD"), Arguments.of("\uD800A", "\uFFFDA"), Arguments.of("A\uDC00", "A\uFFFD")); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPath(String input, String expected) { + Encoder decoder = new Utf8Encoder(); + CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + decoder.encode(cbuf, bbuf, true); + + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPathChuncked(String input, String expected) { + Encoder decoder = new Utf8Encoder(); + char[] chars = input.toCharArray(); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + for (int i = 0; i < chars.length; i++) { + CharBuffer cbuf = CharBuffer.wrap(new char[] { chars[i] }); + decoder.encode(cbuf, bbuf, i == chars.length - 1); + } + + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPath(String input, String expected) { + Encoder encoder = new Utf8Encoder(); + CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + encoder.encode(cbuf, bbuf, true); + + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPathChuncked(String input, String expected) { + Encoder decoder = new Utf8Encoder(); + char[] chars = input.toCharArray(); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + for (int i = 0; i < chars.length; i++) { + CharBuffer cbuf = CharBuffer.wrap(new char[] { chars[i] }).asReadOnlyBuffer(); + decoder.encode(cbuf, bbuf, i == chars.length - 1); + } + + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + +} From 533f380f8b343b7ae10f1978b93df47552495400 Mon Sep 17 00:00:00 2001 From: Unknow Date: Tue, 31 Mar 2026 13:43:43 +0200 Subject: [PATCH 20/31] fix some error --- .../unknow/server/bench/EncoderDecoder.java | 36 ++++++++++++++----- .../java/unknow/server/util/Utf8Decoder.java | 2 +- .../java/unknow/server/util/Utf8Encoder.java | 8 +---- .../server/util/io/ByteBufferReader.java | 27 ++++++++++++++ 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index d14b8e80..c547d26b 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -106,7 +106,7 @@ public class EncoderDecoder { @State(Scope.Thread) public static class Data { - @Param({ "latin", "simple", "complex" }) + @Param({ "latin", "simple", "complex" }) String name; CharBuffer cbuf; ByteBuffer bytes; @@ -133,12 +133,6 @@ private void init(String str) { } } - public static void main(String[] arg) throws Throwable { - Data data = new Data(); - data.init(COMPLEX); - new EncoderDecoder().decoderUtf8(data); - } - // @Benchmark // public ByteBuffer encoderUtf8Servlet(EncodeData data) { // ByteBuffer b = ByteBuffer.allocate(4096); @@ -177,7 +171,7 @@ public CharBuffer decoderUtf8(Data data) { } @Benchmark - public ByteBuffer encoderUtf8Slow(Data data) { + public ByteBuffer encoderSlowUtf8(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); Encoder e = Encoder.from(StandardCharsets.UTF_8); CharBuffer cbuf = data.cbuf.asReadOnlyBuffer(); @@ -189,7 +183,7 @@ public ByteBuffer encoderUtf8Slow(Data data) { } @Benchmark - public CharBuffer decoderUtf8Slow(Data data) { + public CharBuffer decoderSlowUtf8(Data data) { CharBuffer c = CharBuffer.allocate(4096); Decoder d = Decoder.from(StandardCharsets.UTF_8); ByteBuffer bbuf = data.bytes.asReadOnlyBuffer(); @@ -223,4 +217,28 @@ public CharBuffer decoderCharset(Data data) { } return c.clear(); } + + @Benchmark + public ByteBuffer encoderSlowCharset(Data data) { + ByteBuffer b = ByteBuffer.allocate(4096); + CharsetEncoder e = StandardCharsets.UTF_8.newEncoder(); + CharBuffer cbuf = data.cbuf.asReadOnlyBuffer(); + while (cbuf.hasRemaining()) { + e.encode(cbuf, b, false); + b.clear(); + } + return b.clear(); + } + + @Benchmark + public CharBuffer decoderSlowCharset(Data data) { + CharBuffer c = CharBuffer.allocate(4096); + CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); + ByteBuffer bbuf = data.bytes.asReadOnlyBuffer(); + while (bbuf.hasRemaining()) { + d.decode(bbuf, c, false); + c.clear(); + } + return c.clear(); + } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index d6f84859..4ef7c70f 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -30,7 +30,6 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { int cpos = cbuf.position() + cbuf.arrayOffset(); int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); - int maxAscii = Math.min(clim, cpos + blim - bpos); int codePoint = this.codePoint; int r = this.r; @@ -89,6 +88,7 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { int b = barr[bpos++] & 0xFF; if (b < 0x80) { // 1-byte ASCII carr[cpos++] = (char) b; + int maxAscii = Math.min(clim, cpos + blim - bpos); while (cpos < maxAscii && barr[bpos] >= 0) carr[cpos++] = (char) barr[bpos++]; } else if (b < 0xc0) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index 814081be..d4813ae4 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -41,13 +41,11 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { } surrogate = 0; } - int maxAscii = Math.min(clim, cpos + blim - bpos); - while (cpos < maxAscii && carr[cpos] <= 0x7f) - barr[bpos++] = (byte) carr[cpos++]; while (cpos < clim && bpos < blim) { int code = carr[cpos++]; if (code <= 0x7f) { barr[bpos++] = (byte) code; + int maxAscii = Math.min(clim, cpos + blim - bpos); while (cpos < maxAscii && carr[cpos] <= 0x7f) barr[bpos++] = (byte) carr[cpos++]; } else if (code <= 0x7FF) { @@ -103,10 +101,6 @@ private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { slowAppend(0xFFFD, bbuf); surrogate = 0; } - int cpos = bbuf.position(); - int maxAscii = Math.min(cbuf.limit(), cpos + bbuf.remaining()); - while (cpos < maxAscii && cbuf.get(cpos++) <= 0x7f) - bbuf.put((byte) cbuf.get()); while (cbuf.hasRemaining() && bbuf.remaining() >= 4) { int code = cbuf.get(); int i = code >> 10; diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java index 6ef8ec71..bff41d55 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java @@ -48,6 +48,33 @@ public int read(char[] cbuf, int off, int len) throws IOException { return len; } + @Override + public int read(CharBuffer target) throws IOException { + if (in.isOef()) + return -1; + decode(); + int l = buf.remaining(); + int r = target.remaining(); + if (r > l) { + target.put(buf); + return l; + } + l = buf.limit(); + target.put(buf.limit(buf.position() + r)); + buf.limit(l); + return r; + } + + @Override + public long skip(long n) throws IOException { + if (in.isOef()) + return 0; + decode(); + int l = (int) Math.min(n, buf.remaining()); + buf.position(buf.position() + l); + return n; + } + @Override public void close() throws IOException { in.close(); From 0abc781d1136c5076a6f5f4626361f60f3b14e57 Mon Sep 17 00:00:00 2001 From: Unknow Date: Wed, 1 Apr 2026 00:06:30 +0200 Subject: [PATCH 21/31] update utf8 decoder --- .../unknow/server/bench/EncoderDecoder.java | 28 +++++++++---------- .../unknow/server/util/Utf8DecoderTest.java | 6 ++-- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index c547d26b..f600eb87 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -171,10 +171,10 @@ public CharBuffer decoderUtf8(Data data) { } @Benchmark - public ByteBuffer encoderSlowUtf8(Data data) { + public ByteBuffer encoderCharset(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); - Encoder e = Encoder.from(StandardCharsets.UTF_8); - CharBuffer cbuf = data.cbuf.asReadOnlyBuffer(); + CharsetEncoder e = StandardCharsets.UTF_8.newEncoder(); + CharBuffer cbuf = data.cbuf.duplicate(); while (cbuf.hasRemaining()) { e.encode(cbuf, b, false); b.clear(); @@ -183,10 +183,10 @@ public ByteBuffer encoderSlowUtf8(Data data) { } @Benchmark - public CharBuffer decoderSlowUtf8(Data data) { + public CharBuffer decoderCharset(Data data) { CharBuffer c = CharBuffer.allocate(4096); - Decoder d = Decoder.from(StandardCharsets.UTF_8); - ByteBuffer bbuf = data.bytes.asReadOnlyBuffer(); + CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); + ByteBuffer bbuf = data.bytes.duplicate(); while (bbuf.hasRemaining()) { d.decode(bbuf, c, false); c.clear(); @@ -195,10 +195,10 @@ public CharBuffer decoderSlowUtf8(Data data) { } @Benchmark - public ByteBuffer encoderCharset(Data data) { + public ByteBuffer slowEncoderUtf8(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); - CharsetEncoder e = StandardCharsets.UTF_8.newEncoder(); - CharBuffer cbuf = data.cbuf.duplicate(); + Encoder e = Encoder.from(StandardCharsets.UTF_8); + CharBuffer cbuf = data.cbuf.asReadOnlyBuffer(); while (cbuf.hasRemaining()) { e.encode(cbuf, b, false); b.clear(); @@ -207,10 +207,10 @@ public ByteBuffer encoderCharset(Data data) { } @Benchmark - public CharBuffer decoderCharset(Data data) { + public CharBuffer slowDecoderUtf8(Data data) { CharBuffer c = CharBuffer.allocate(4096); - CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); - ByteBuffer bbuf = data.bytes.duplicate(); + Decoder d = Decoder.from(StandardCharsets.UTF_8); + ByteBuffer bbuf = data.bytes.asReadOnlyBuffer(); while (bbuf.hasRemaining()) { d.decode(bbuf, c, false); c.clear(); @@ -219,7 +219,7 @@ public CharBuffer decoderCharset(Data data) { } @Benchmark - public ByteBuffer encoderSlowCharset(Data data) { + public ByteBuffer slowEncoderCharset(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); CharsetEncoder e = StandardCharsets.UTF_8.newEncoder(); CharBuffer cbuf = data.cbuf.asReadOnlyBuffer(); @@ -231,7 +231,7 @@ public ByteBuffer encoderSlowCharset(Data data) { } @Benchmark - public CharBuffer decoderSlowCharset(Data data) { + public CharBuffer slowDecoderCharset(Data data) { CharBuffer c = CharBuffer.allocate(4096); CharsetDecoder d = StandardCharsets.UTF_8.newDecoder(); ByteBuffer bbuf = data.bytes.asReadOnlyBuffer(); diff --git a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java index 442828a1..5c52a968 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java @@ -17,9 +17,11 @@ static Stream utf8Strings() { } static Stream invalidUtf8() { - return Stream.of(new byte[] { (byte) 0x80 }, new byte[] { (byte) 0xC0 }, new byte[] { (byte) 0xE0, (byte) 0x80 }, + return Stream.of(new byte[] { (byte) 0x80 }, new byte[] { (byte) 0xC0 }, new byte[] { (byte) 0xE0, (byte) 0x80 }, //new byte[] { (byte) 0xF0, (byte) 0x80, (byte) 0x80 }, - new byte[] { (byte) 0xC0, (byte) 0xAF }, new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, new byte[] { (byte) 0xF8 }, new byte[] { (byte) 0xFF }); + //new byte[] { (byte) 0xC0, (byte) 0xAF }, + // new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, + new byte[] { (byte) 0xF8 }, new byte[] { (byte) 0xFF }); } @ParameterizedTest From b251ce892d2fa67c8ecad101f580aa1103d5deb8 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 2 Apr 2026 00:28:24 +0200 Subject: [PATCH 22/31] update decoder fast path --- .../java/unknow/server/util/Utf8Decoder.java | 289 ++++++++---------- .../unknow/server/util/Utf8DecoderTest.java | 6 +- 2 files changed, 130 insertions(+), 165 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 4ef7c70f..9b1ecac2 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -1,12 +1,20 @@ package unknow.server.util; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.CharBuffer; public class Utf8Decoder implements Decoder { - /** code point in building */ - private int codePoint; + private static final VarHandle LONG = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN); + private static final VarHandle INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); + private static final VarHandle SHORT = MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.LITTLE_ENDIAN); + /** code point in building */ + private int cp; + /** min code point allowed */ + private int minCp; /** remaining octet to read */ private int r; @@ -25,170 +33,127 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { byte[] barr = bbuf.array(); int bpos = bbuf.position() + bbuf.arrayOffset(); int blim = bbuf.limit() + bbuf.arrayOffset(); - char[] carr = cbuf.array(); int cpos = cbuf.position() + cbuf.arrayOffset(); int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); - - int codePoint = this.codePoint; - int r = this.r; - if (r > 0) { - if (bpos + r <= blim) { // can read all - int b; - switch (r) { - case 3: - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint < 0x10) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - case 2: - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint < 0x20) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - case 1: - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); - } - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - carr[cpos++] = (char) ((cp >> 10) + 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - carr[cpos++] = (char) codePoint; - r = 0; - } else { - while (bpos < blim && cpos < clim) { - int b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: " + b); - codePoint = (codePoint << 6) | (b & 0x3F); - if ((r == 2 && codePoint < 0x20) || (r == 3 && codePoint < 0x10)) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - if (--r == 0) { - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - carr[cpos++] = (char) ((cp >> 10) + 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - carr[cpos++] = (char) codePoint; - } - } - } + remainingArray(bbuf, bpos, cbuf, cpos, cp, r); + if (r > 0) + return; + bpos = bbuf.position() + bbuf.arrayOffset(); + cpos = cbuf.position() + cbuf.arrayOffset(); + } + int max = Math.min(clim, cpos + blim - bpos) - 8; + while (cpos < max) { + long l = (long) LONG.get(barr, bpos); + if ((l & 0x8080808080808080L) != 0L) + break; + carr[cpos++] = (char) l; + carr[cpos++] = (char) (l >>> 8); + carr[cpos++] = (char) (l >>> 16); + carr[cpos++] = (char) (l >>> 24); + carr[cpos++] = (char) (l >>> 32); + carr[cpos++] = (char) (l >>> 40); + carr[cpos++] = (char) (l >>> 48); + carr[cpos++] = (char) (l >>> 56); + bpos += 8; } + int cp; + loop: while (bpos < blim && cpos < clim) { + int b = barr[bpos++]; - while (bpos < blim && cpos < clim) { - int b = barr[bpos++] & 0xFF; - if (b < 0x80) { // 1-byte ASCII + if (b >= 0) carr[cpos++] = (char) b; - int maxAscii = Math.min(clim, cpos + blim - bpos); - while (cpos < maxAscii && barr[bpos] >= 0) - carr[cpos++] = (char) barr[bpos++]; - } else if (b < 0xc0) - throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); - else { - if (b < 0xe0) { - codePoint = b & 0x1F; - if (codePoint < 2) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - if (bpos < blim) { - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - carr[cpos++] = (char) ((cp >> 10) + 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - carr[cpos++] = (char) codePoint; - } else { - r = 1; - break; - } - } else if (b < 0xf0) { - codePoint = b & 0x0F; - if (bpos + 1 < blim) { - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint < 0x20) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - carr[cpos++] = (char) ((cp >> 10) + 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - carr[cpos++] = (char) codePoint; - } else { - r = 2; - break; - } - } else if (b < 0xf4) { - codePoint = b & 0x07; - if (bpos + 2 < blim) { - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint < 0x10) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - carr[cpos++] = (char) ((cp >> 10) + 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - carr[cpos++] = (char) codePoint; - } else { - r = 3; - break; - } + else if (b < -64) + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b & 0xFF, 16)); + else if (b < -32) { + cp = b & 0x1F; + if (bpos == blim) { + this.cp = cp; + r = 1; + minCp = 0x80; + break loop; + } + cp = continuation(cp, barr[bpos++]); + if (cp < 0x80) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + carr[cpos++] = (char) cp; + } else if (b < -16) { + cp = b & 0x0F; + if (bpos + 1 >= blim) { + minCp = 0x800; + remainingArray(bbuf, bpos, cbuf, cpos, cp, 2); + return; + } + short s = (short) SHORT.get(barr, bpos); + if ((s & 0xc0c0) != 0x8080) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + + cp = (cp << 12) ^ ((s & 0x3F) << 6) ^ ((s >> 8) & 0x3F); + if (cp < 0x800) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + carr[cpos++] = (char) cp; + bpos += 2; + } else if (b < -8) { + cp = b & 0x07; + if (bpos + 3 >= blim) { + minCp = 0x10000; + remainingArray(bbuf, bpos, cbuf, cpos, cp, 3); + return; + } + int i = (int) INT.get(barr, bpos); + if ((i & 0x00C0C0C0) != 0x00808080) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + cp = (cp << 18) ^ ((i & 0x3F) << 12) ^ (((i >> 8) & 0x3F) << 6) ^ (((i >> 16) & 0x3F)); + if (cp < 0x10000) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + if (cp > 0xFFFF) { // Surrogate pair + cp -= 0x10000; + carr[cpos++] = (char) ((cp >> 10) ^ 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) ^ 0xDC00); } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); - } + carr[cpos++] = (char) cp; + bpos += 3; + } else + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } + updatePos(bbuf, bpos, cbuf, cpos); + } - if (r > 0) { - while (bpos < blim && cpos < clim) { - int b = barr[bpos++] & 0xFF; - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if (--r == 0) { - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; - carr[cpos++] = (char) ((cp >> 10) + 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) + 0xDC00); - } else - carr[cpos++] = (char) codePoint; - } + private void remainingArray(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos, int cp, int r) { + byte[] barr = bbuf.array(); + int blim = bbuf.limit() + bbuf.arrayOffset(); + char[] carr = cbuf.array(); + int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); + + while (bpos < blim && cpos < clim) { + cp = continuation(cp, barr[bpos++]); + if (--r == 0) { + if (cp < minCp) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence " + Integer.toString(cp, 16) + " < " + Integer.toString(minCp, 16)); + if (cp > 0xFFFF) { // Surrogate pair + cp -= 0x10000; + carr[cpos++] = (char) ((cp >> 10) ^ 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) ^ 0xDC00); + } else + carr[cpos++] = (char) cp; + this.r = r; + updatePos(bbuf, bpos, cbuf, cpos); + return; } } - + this.cp = cp; this.r = r; - this.codePoint = codePoint; + updatePos(bbuf, bpos, cbuf, cpos); + } + + private static int continuation(int cp, int b) { + if ((b & 0xc0) != 0x80) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + return (cp << 6) ^ (b & 0x3F); + } + private static void updatePos(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos) { bbuf.position(bpos - bbuf.arrayOffset()); cbuf.position(cpos - cbuf.arrayOffset()); } @@ -204,8 +169,8 @@ private void slowAppend(int b, CharBuffer cbuf) { if ((b >> 6) != 0b10) throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - codePoint = (codePoint << 6) | (b & 0x3F); - if ((r == 2 && codePoint < 0x20) || (r == 3 && codePoint < 0x10)) + cp = (cp << 6) | (b & 0x3F); + if ((r == 2 && cp < 0x20) || (r == 3 && cp < 0x10)) throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); if (--r == 0) slowAppend(cbuf); @@ -215,25 +180,25 @@ else if (b < 0xc0) throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); else if (b < 0xe0) { r = 1; - codePoint = b & 0x1F; - if (codePoint < 2) + cp = b & 0x1F; + if (cp < 2) throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); } else if (b < 0xf0) { r = 2; - codePoint = b & 0x0F; + cp = b & 0x0F; } else if (b < 0xfb) { r = 3; - codePoint = b & 0x07; + cp = b & 0x07; } else throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } private void slowAppend(CharBuffer cbuf) { - if (codePoint > 0xFFFF) { // Surrogate pair - int cp = codePoint - 0x10000; + if (cp > 0xFFFF) { // Surrogate pair + cp -= 0x10000; cbuf.put((char) ((cp >> 10) + 0xD800)).put((char) ((cp & 0x3FF) + 0xDC00)); } else - cbuf.put((char) codePoint); - codePoint = 0; + cbuf.put((char) cp); + cp = 0; } } diff --git a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java index 5c52a968..a966366c 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java @@ -18,9 +18,9 @@ static Stream utf8Strings() { static Stream invalidUtf8() { return Stream.of(new byte[] { (byte) 0x80 }, new byte[] { (byte) 0xC0 }, new byte[] { (byte) 0xE0, (byte) 0x80 }, - //new byte[] { (byte) 0xF0, (byte) 0x80, (byte) 0x80 }, - //new byte[] { (byte) 0xC0, (byte) 0xAF }, - // new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, + new byte[] { (byte) 0xF0, (byte) 0x80, (byte) 0x80 }, + new byte[] { (byte) 0xC0, (byte) 0xAF }, + new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, new byte[] { (byte) 0xF8 }, new byte[] { (byte) 0xFF }); } From 58f1a05b6f06091e0149bc16acb958afe4e41bb6 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 2 Apr 2026 22:16:00 +0200 Subject: [PATCH 23/31] update --- unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java | 1 + 1 file changed, 1 insertion(+) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java index 49a2ba65..11e5c9e0 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorker.java @@ -173,6 +173,7 @@ protected final void selected(long now, SelectionKey key) throws IOException { if (co.next != co) logger.error("failed to write {}", co, e); startClose(co, now); + key.cancel(); } finally { buf.clear(); } From 60236035e6c3f84a7500a7befe6f99fba4d36319 Mon Sep 17 00:00:00 2001 From: Unknow Date: Thu, 2 Apr 2026 23:31:02 +0200 Subject: [PATCH 24/31] up --- .../main/java/unknow/server/util/Utf8Decoder.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 9b1ecac2..123370c3 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -49,14 +49,15 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { if ((l & 0x8080808080808080L) != 0L) break; carr[cpos++] = (char) l; - carr[cpos++] = (char) (l >>> 8); - carr[cpos++] = (char) (l >>> 16); - carr[cpos++] = (char) (l >>> 24); - carr[cpos++] = (char) (l >>> 32); - carr[cpos++] = (char) (l >>> 40); - carr[cpos++] = (char) (l >>> 48); - carr[cpos++] = (char) (l >>> 56); + carr[cpos ++] = (char) (l >>> 8); + carr[cpos ++] = (char) (l >>> 16); + carr[cpos ++] = (char) (l >>> 24); + carr[cpos ++] = (char) (l >>> 32); + carr[cpos ++] = (char) (l >>> 40); + carr[cpos ++] = (char) (l >>> 48); + carr[cpos ++] = (char) (l >>> 56); bpos += 8; +// cpos += 8; } int cp; loop: while (bpos < blim && cpos < clim) { From fa4fc55e0756c19672b93d59b1aa2bd14ba60b86 Mon Sep 17 00:00:00 2001 From: Unknow Date: Sun, 12 Apr 2026 12:31:56 +0200 Subject: [PATCH 25/31] update encoder/decoder --- .../server/servlet/impl/ServletWriter.java | 2 + .../main/java/unknow/server/util/Decoder.java | 46 ++++- .../main/java/unknow/server/util/Encoder.java | 49 ++++- .../java/unknow/server/util/Utf8Decoder.java | 181 +++++++++++----- .../java/unknow/server/util/Utf8Encoder.java | 195 ++++++++++-------- .../server/util/io/ByteBufferReader.java | 22 +- .../java/unknow/server/util/DecoderTest.java | 132 ++++++++++++ .../java/unknow/server/util/EncoderTest.java | 85 ++++++++ .../unknow/server/util/Utf8DecoderTest.java | 133 +----------- .../unknow/server/util/Utf8EncoderTest.java | 73 +------ 10 files changed, 566 insertions(+), 352 deletions(-) create mode 100644 unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java create mode 100644 unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java index 7010f788..60d0e92d 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/impl/ServletWriter.java @@ -60,6 +60,8 @@ public void flush() throws IOException { @Override public void close() throws IOException { + while (enc.flush(out.buffer)) + out.flush(); out.close(); } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Decoder.java index f8d90b02..1654cbd4 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Decoder.java @@ -8,7 +8,11 @@ import java.nio.charset.StandardCharsets; public interface Decoder { - + /** + * create a nex decoder for a charset + * @param charset the charset + * @return the decoder + */ public static Decoder from(Charset charset) { if (charset.equals(StandardCharsets.UTF_8)) return new Utf8Decoder(); @@ -23,18 +27,56 @@ public static Decoder from(Charset charset) { */ void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput); + /** + * flush remaining char + * @param cbuf output + * @return true if we need to recall flush with more space + */ + boolean flush(CharBuffer cbuf); + public class DefaultDecoder implements Decoder { private final CharsetDecoder dec; + private final ByteBuffer b; + private boolean endCalled; public DefaultDecoder(CharsetDecoder dec) { this.dec = dec; + this.b = ByteBuffer.allocate(4096); + this.endCalled = false; } @Override public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { - CoderResult r = dec.decode(bbuf, cbuf, endOfInput); + int l = bbuf.limit(); + while (bbuf.position() < l) { + b.put(bbuf.limit(Math.min(l, bbuf.position() + b.remaining()))); + CoderResult r = dec.decode(b.flip(), cbuf, endOfInput); + b.compact(); + bbuf.limit(l); + if (endOfInput) + endCalled = true; + if (r.isOverflow()) + return; + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } + } + + @Override + public boolean flush(CharBuffer cbuf) { + if (b.position() > 0 || !endCalled) { + CoderResult r = dec.decode(b.flip(), cbuf, true); + b.compact(); + endCalled = true; + if (r.isOverflow()) + return true; + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } + CoderResult r = dec.flush(cbuf); if (r.isError()) throw new IllegalArgumentException(r.toString()); + return r.isOverflow(); } } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Encoder.java index 5c837dc6..b2164d71 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Encoder.java @@ -5,14 +5,21 @@ import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; public interface Encoder { + static final byte[] REPL = { (byte) 0xEF, (byte) 0xBF, (byte) 0xBD }; + /** + * get encoder from a charset + * @param charset the charset + * @return the encoder + */ public static Encoder from(Charset charset) { if (charset.equals(StandardCharsets.UTF_8)) return new Utf8Encoder(); - return new DefaultEncoder(charset.newEncoder()); + return new DefaultEncoder(charset.newEncoder().replaceWith(REPL).onMalformedInput(CodingErrorAction.REPLACE)); } /** @@ -23,18 +30,56 @@ public static Encoder from(Charset charset) { */ void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput); + /** + * flush remaining bytes + * @param bbuf output + * @return true if we need to recall flush with more space + */ + boolean flush(ByteBuffer bbuf); + public class DefaultEncoder implements Encoder { private final CharsetEncoder enc; + private final CharBuffer c; + private boolean endCalled; public DefaultEncoder(CharsetEncoder dec) { this.enc = dec; + this.c = CharBuffer.allocate(4096); + this.endCalled = false; } @Override public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { - CoderResult r = enc.encode(cbuf, bbuf, endOfInput); + int l = cbuf.limit(); + while (cbuf.position() < l) { + c.put(cbuf.limit(Math.min(l, cbuf.position() + c.remaining()))); + CoderResult r = enc.encode(c.flip(), bbuf, endOfInput); + c.compact(); + cbuf.limit(l); + if (endOfInput) + endCalled = true; + if (r.isOverflow()) + return; + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } + } + + @Override + public boolean flush(ByteBuffer bbuf) { + if (c.position() > 0 || !endCalled) { + CoderResult r = enc.encode(c.flip(), bbuf, true); + c.compact(); + endCalled = true; + if (r.isOverflow()) + return true; + if (r.isError()) + throw new IllegalArgumentException(r.toString()); + } + CoderResult r = enc.flush(bbuf); if (r.isError()) throw new IllegalArgumentException(r.toString()); + return r.isOverflow(); } } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 123370c3..3db045a9 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -23,12 +23,17 @@ public void decode(ByteBuffer bbuf, CharBuffer cbuf, boolean endOfInput) { if (bbuf.hasArray() && cbuf.hasArray()) fastDecode(bbuf, cbuf); else { - slowDecode(bbuf, cbuf); + slowDecode(bbuf.order(ByteOrder.LITTLE_ENDIAN), cbuf); } if (endOfInput && !bbuf.hasRemaining() && r > 0) throw new IllegalArgumentException("Invalid UTF-8 string, truncated continuation bytes"); } + @Override + public boolean flush(CharBuffer cbuf) { + return false; + } + private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { byte[] barr = bbuf.array(); int bpos = bbuf.position() + bbuf.arrayOffset(); @@ -48,36 +53,40 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { long l = (long) LONG.get(barr, bpos); if ((l & 0x8080808080808080L) != 0L) break; - carr[cpos++] = (char) l; - carr[cpos ++] = (char) (l >>> 8); - carr[cpos ++] = (char) (l >>> 16); - carr[cpos ++] = (char) (l >>> 24); - carr[cpos ++] = (char) (l >>> 32); - carr[cpos ++] = (char) (l >>> 40); - carr[cpos ++] = (char) (l >>> 48); - carr[cpos ++] = (char) (l >>> 56); + carr[cpos] = (char) l; + carr[cpos + 1] = (char) (l >>> 8); + carr[cpos + 2] = (char) (l >>> 16); + carr[cpos + 3] = (char) (l >>> 24); + carr[cpos + 4] = (char) (l >>> 32); + carr[cpos + 5] = (char) (l >>> 40); + carr[cpos + 6] = (char) (l >>> 48); + carr[cpos + 7] = (char) (l >>> 56); bpos += 8; -// cpos += 8; + cpos += 8; } int cp; - loop: while (bpos < blim && cpos < clim) { + while (bpos < blim && cpos < clim) { int b = barr[bpos++]; if (b >= 0) carr[cpos++] = (char) b; else if (b < -64) - throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b & 0xFF, 16)); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 start byte: 0x" + Integer.toString(b & 0xFF, 16)); else if (b < -32) { cp = b & 0x1F; if (bpos == blim) { this.cp = cp; r = 1; minCp = 0x80; - break loop; + updatePos(bbuf, bpos, cbuf, cpos); + return; } - cp = continuation(cp, barr[bpos++]); + b = barr[bpos++]; + if ((b & 0xc0) != 0x80) + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); + cp = (cp << 6) ^ (b & 0x3F); if (cp < 0x80) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence"); carr[cpos++] = (char) cp; } else if (b < -16) { cp = b & 0x0F; @@ -88,11 +97,11 @@ else if (b < -32) { } short s = (short) SHORT.get(barr, bpos); if ((s & 0xc0c0) != 0x8080) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); cp = (cp << 12) ^ ((s & 0x3F) << 6) ^ ((s >> 8) & 0x3F); if (cp < 0x800) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence"); carr[cpos++] = (char) cp; bpos += 2; } else if (b < -8) { @@ -104,10 +113,10 @@ else if (b < -32) { } int i = (int) INT.get(barr, bpos); if ((i & 0x00C0C0C0) != 0x00808080) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); cp = (cp << 18) ^ ((i & 0x3F) << 12) ^ (((i >> 8) & 0x3F) << 6) ^ (((i >> 16) & 0x3F)); if (cp < 0x10000) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence"); if (cp > 0xFFFF) { // Surrogate pair cp -= 0x10000; carr[cpos++] = (char) ((cp >> 10) ^ 0xD800); @@ -116,7 +125,7 @@ else if (b < -32) { carr[cpos++] = (char) cp; bpos += 3; } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } updatePos(bbuf, bpos, cbuf, cpos); } @@ -128,10 +137,13 @@ private void remainingArray(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); while (bpos < blim && cpos < clim) { - cp = continuation(cp, barr[bpos++]); + int b = barr[bpos++]; + if ((b & 0xc0) != 0x80) + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); + cp = (cp << 6) ^ (b & 0x3F); if (--r == 0) { if (cp < minCp) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence " + Integer.toString(cp, 16) + " < " + Integer.toString(minCp, 16)); + error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence " + Integer.toString(cp, 16) + " < " + Integer.toString(minCp, 16)); if (cp > 0xFFFF) { // Surrogate pair cp -= 0x10000; carr[cpos++] = (char) ((cp >> 10) ^ 0xD800); @@ -148,10 +160,9 @@ private void remainingArray(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos updatePos(bbuf, bpos, cbuf, cpos); } - private static int continuation(int cp, int b) { - if ((b & 0xc0) != 0x80) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); - return (cp << 6) ^ (b & 0x3F); + private static void error(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos, String msg) { + updatePos(bbuf, bpos, cbuf, cpos); + throw new IllegalArgumentException(msg); } private static void updatePos(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos) { @@ -159,47 +170,107 @@ private static void updatePos(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cp cbuf.position(cpos - cbuf.arrayOffset()); } + private final char[] CARR = new char[8]; + private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { + if (r > 0 && slowRemaining(bbuf, cbuf, cp, r)) + return; + + int max = Math.min(bbuf.limit(), bbuf.position() + cbuf.remaining()) - 8; + int bpos = bbuf.position(); + while (bpos < max) { + long l = bbuf.getLong(bpos); + if ((l & 0x8080808080808080L) != 0L) + break; + CARR[0] = (char) l; + CARR[1] = (char) (l >>> 8); + CARR[2] = (char) (l >>> 16); + CARR[3] = (char) (l >>> 24); + CARR[4] = (char) (l >>> 32); + CARR[5] = (char) (l >>> 40); + CARR[6] = (char) (l >>> 48); + CARR[7] = (char) (l >>> 56); + cbuf.put(CARR); + bpos += 8; + } + int cp; while (bbuf.hasRemaining() && cbuf.remaining() > 1) { - slowAppend(bbuf.get() & 0xFF, cbuf); + int b = bbuf.get(); + if (b >= 0) + cbuf.put((char) b); + else if (b < -64) + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b & 0xFF, 16)); + else if (b < -32) { + cp = b & 0x1F; + if (!bbuf.hasRemaining()) { + minCp = 0x80; + r = 1; + this.cp = cp; + return; + } + b = bbuf.get(); + if ((b & 0xc0) != 0x80) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + cp = (cp << 6) ^ (b & 0x3F); + if (cp < 0x80) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + cbuf.put((char) cp); + } else if (b < -16) { + cp = b & 0x0F; + if (bbuf.remaining() < 2) { + minCp = 0x800; + slowRemaining(bbuf, cbuf, cp, 2); + return; + } + short s = bbuf.getShort(); + if ((s & 0xc0c0) != 0x8080) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + + cp = (cp << 12) ^ ((s & 0x3F) << 6) ^ ((s >> 8) & 0x3F); + if (cp < 0x800) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + cbuf.put((char) cp); + } else if (b < -8) { + cp = b & 0x07; + if (bbuf.remaining() < 4) { + minCp = 0x10000; + slowRemaining(bbuf, cbuf, cp, 3); + return; + } + int i = bbuf.getInt(bbuf.position()); + bbuf.position(bbuf.position() + 3); + if ((i & 0x00C0C0C0) != 0x00808080) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + cp = (cp << 18) ^ ((i & 0x3F) << 12) ^ (((i >> 8) & 0x3F) << 6) ^ (((i >> 16) & 0x3F)); + if (cp < 0x10000) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + slowAppend(cbuf, cp); + } else + throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } } - private void slowAppend(int b, CharBuffer cbuf) { - if (r > 0) { - if ((b >> 6) != 0b10) - throw new IllegalArgumentException("Invalid UTF-8 continuation byte: 0x" + Integer.toString(b, 16)); - - cp = (cp << 6) | (b & 0x3F); - if ((r == 2 && cp < 0x20) || (r == 3 && cp < 0x10)) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); + private boolean slowRemaining(ByteBuffer bbuf, CharBuffer cbuf, int cp, int r) { + while (bbuf.hasRemaining() && cbuf.remaining() > 1) { + int b = bbuf.get(); + if ((b & 0xc0) != 0x80) + throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); + cp = (cp << 6) ^ (b & 0x3F); if (--r == 0) - slowAppend(cbuf); - } else if (b < 0x80) // 1-byte ASCII - cbuf.put((char) b); - else if (b < 0xc0) - throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); - else if (b < 0xe0) { - r = 1; - cp = b & 0x1F; - if (cp < 2) - throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - } else if (b < 0xf0) { - r = 2; - cp = b & 0x0F; - } else if (b < 0xfb) { - r = 3; - cp = b & 0x07; - } else - throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); + slowAppend(cbuf, cp); + } + this.cp = cp; + this.r = r; + return r > 0; } - private void slowAppend(CharBuffer cbuf) { + private void slowAppend(CharBuffer cbuf, int cp) { + if (cp < minCp) + throw new IllegalArgumentException("Invalid UTF-8 overlong sequence " + Integer.toString(cp, 16) + " < " + Integer.toString(minCp, 16)); if (cp > 0xFFFF) { // Surrogate pair cp -= 0x10000; cbuf.put((char) ((cp >> 10) + 0xD800)).put((char) ((cp & 0x3FF) + 0xDC00)); } else cbuf.put((char) cp); - cp = 0; } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index d4813ae4..55f5d38d 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -1,10 +1,17 @@ package unknow.server.util; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.CharBuffer; public class Utf8Encoder implements Encoder { - private int surrogate; + private static final VarHandle INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); + + private static final int repl = 0x00BDBFEF; + + private int cp; @Override public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { @@ -13,7 +20,18 @@ public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { if (cbuf.hasArray() && bbuf.hasArray()) fastEncode(cbuf, bbuf, endOfInput); else - slowEncode(cbuf, bbuf, endOfInput); + slowEncode(cbuf, bbuf.order(ByteOrder.LITTLE_ENDIAN), endOfInput); + } + + @Override + public boolean flush(ByteBuffer bbuf) { + if (cp != 0) { + if (bbuf.remaining() < 3) + return true; + bbuf.put(REPL); + cp = 0; + } + return false; } private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { @@ -25,66 +43,70 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { int bpos = bbuf.position() + bbuf.arrayOffset(); int blim = bbuf.limit() - 3 + bbuf.arrayOffset(); - if (surrogate != 0) { + if (cp != 0) { int low = carr[cpos]; if (low >= 0xDC00 && low <= 0xDFFF) { cpos++; - int code = 0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00); - barr[bpos++] = (byte) (0xF0 | (code >> 18)); - barr[bpos++] = (byte) (0x80 | ((code >> 12) & 0x3F)); - barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + int c = 0x10000 + ((cp & 0x7FF) << 10) + (low & 0x3FF); + c = 0x808080F0 | (c >> 18) | ((c >> 4) & 0x00003F00) | ((c << 10) & 0x003F0000) | (c << 24 & 0x3F000000); + INT.set(barr, bpos, c); + bpos += 4; } else { - barr[bpos++] = (byte) 0xEF; - barr[bpos++] = (byte) 0xBF; - barr[bpos++] = (byte) 0xbd; + INT.set(barr, bpos, repl); + bpos += 3; } - surrogate = 0; + cp = 0; } + + int max = Math.min(clim, cpos + blim - bpos); + int start = cpos; + while (cpos < max && carr[cpos] < 0x80) + cpos++; + int len = cpos - start; + for (int i = 0; i < len; i++) + barr[bpos + i] = (byte) carr[start + i]; + bpos += len; + while (cpos < clim && bpos < blim) { int code = carr[cpos++]; - if (code <= 0x7f) { + if (code < 0x80) { barr[bpos++] = (byte) code; - int maxAscii = Math.min(clim, cpos + blim - bpos); - while (cpos < maxAscii && carr[cpos] <= 0x7f) - barr[bpos++] = (byte) carr[cpos++]; - } else if (code <= 0x7FF) { - barr[bpos++] = (byte) (0xC0 | (code >> 6)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); - } else { - int i = code >> 10; - if (i == 0b110110) { // high surrogate - if (cpos < clim) { - int low = carr[cpos]; - if (low >= 0xDC00 && low <= 0xDFFF) { - cpos++; - int c = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); - barr[bpos++] = (byte) (0xF0 | (c >> 18)); - barr[bpos++] = (byte) (0x80 | ((c >> 12) & 0x3F)); - barr[bpos++] = (byte) (0x80 | ((c >> 6) & 0x3F)); - barr[bpos++] = (byte) (0x80 | (c & 0x3F)); - } else { - barr[bpos++] = (byte) 0xEF; - barr[bpos++] = (byte) 0xBF; - barr[bpos++] = (byte) 0xbd; - } - } else if (endOfInput) { - barr[bpos++] = (byte) 0xEF; - barr[bpos++] = (byte) 0xBF; - barr[bpos++] = (byte) 0xbd; + } else if (code < 0x800) { + int c = 0x80C0 | (code >> 6) | ((code << 8) & 0x3F00); + INT.set(barr, bpos, c); + bpos += 2; + } else if (code < 0xD800) { + int c = 0x008080E0 | (code >> 12) | ((code << 2) & 0x3F00) | (code << 16 & 0x3F0000); + INT.set(barr, bpos, c); + bpos += 3; + } else if (code <= 0xDC00) { + // high surrogate + if (cpos < clim) { + int low = carr[cpos]; + if (low >= 0xDC00 && low <= 0xDFFF) { + cpos++; + int c = 0x10000 + ((code & 0x7FF) << 10) + (low & 0x3FF); + c = 0x808080F0 | (c >> 18) | ((c >> 4) & 0x00003F00) | ((c << 10) & 0x003F0000) | (c << 24 & 0x3F000000); + INT.set(barr, bpos, c); + bpos += 4; } else { - surrogate = code; - break; + INT.set(barr, bpos, repl); + bpos += 3; } - } else if (i == 0b110111) { // lone low surrogate - barr[bpos++] = (byte) 0xEF; - barr[bpos++] = (byte) 0xBF; - barr[bpos++] = (byte) 0xbd; + } else if (endOfInput) { + INT.set(barr, bpos, repl); + bpos += 3; } else { - barr[bpos++] = (byte) (0xE0 | (code >> 12)); - barr[bpos++] = (byte) (0x80 | ((code >> 6) & 0x3F)); - barr[bpos++] = (byte) (0x80 | (code & 0x3F)); + cp = code; + break; } + } else if (code < 0xE000) { // lone low surrogate + INT.set(barr, bpos, repl); + bpos += 3; + } else { + int c = 0x008080E0 | (code >> 12) | ((code << 2) & 0x3F00) | (code << 16 & 0x3F0000); + INT.set(barr, bpos, c); + bpos += 3; } } cbuf.position(cpos - cbuf.arrayOffset()); @@ -92,53 +114,54 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { } private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { - if (surrogate != 0) { - int low = cbuf.get(cbuf.position()); - if (low >= 0xDC00 && low <= 0xDFFF) { - slowAppend(0x10000 + ((surrogate - 0xD800) << 10) + (low - 0xDC00), bbuf); - cbuf.position(cbuf.position() + 1); - } else - slowAppend(0xFFFD, bbuf); - surrogate = 0; + if (cp != 0) { + int cpos = cbuf.position(); + int low = cbuf.get(cpos); + if (slowSurrogate(bbuf, cp, low)) + cbuf.position(cpos + 1); + cp = 0; } while (cbuf.hasRemaining() && bbuf.remaining() >= 4) { int code = cbuf.get(); - int i = code >> 10; - if (i == 0b110110) { // high surrogate + if (code < 0x80) { + bbuf.put((byte) code); + } else if (code < 0x800) { + bbuf.putShort((short) (0x80C0 | (code >> 6) | (code & 0x3F))); + } else if (code < 0xD800) { + int p = bbuf.position(); + int c = 0x008080E0 | (code >> 12) | ((code << 2) & 0x3F00) | (code << 16 & 0x3F0000); + bbuf.putInt(p, c).position(p + 3); + } else if (code <= 0xDC00) { + // high surrogate if (cbuf.hasRemaining()) { - int low = cbuf.get(cbuf.position()); - if (low >= 0xDC00 && low <= 0xDFFF) { - code = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00); - cbuf.position(cbuf.position() + 1); - } else - code = 0xFFFD; + int cpos = cbuf.position(); + int low = cbuf.get(cpos); + if (slowSurrogate(bbuf, code, low)) + cbuf.position(cpos + 1); } else if (endOfInput) - code = 0xFFFD; + bbuf.put(REPL); else { - surrogate = code; - return; + cp = code; + break; } - } else if (i == 0b110111) // lone low surrogate - code = 0xFFFD; - slowAppend(code, bbuf); + } else if (code < 0xE000) // lone low surrogate + bbuf.put(REPL); + else { + int p = bbuf.position(); + int c = 0x008080E0 | (code >> 12) | ((code << 2) & 0x3F00) | (code << 16 & 0x3F0000); + bbuf.putInt(p, c).position(p + 3); + } } } - private void slowAppend(int code, ByteBuffer bbuf) { - if (code <= 0x7F) - bbuf.put((byte) code); - else if (code <= 0x7FF) { - bbuf.put((byte) (0xC0 | (code >> 6))); - bbuf.put((byte) (0x80 | (code & 0x3F))); - } else if (code <= 0xFFFF) { - bbuf.put((byte) (0xE0 | (code >> 12))); - bbuf.put((byte) (0x80 | ((code >> 6) & 0x3F))); - bbuf.put((byte) (0x80 | (code & 0x3F))); - } else { - bbuf.put((byte) (0xF0 | (code >> 18))); - bbuf.put((byte) (0x80 | ((code >> 12) & 0x3F))); - bbuf.put((byte) (0x80 | ((code >> 6) & 0x3F))); - bbuf.put((byte) (0x80 | (code & 0x3F))); + private boolean slowSurrogate(ByteBuffer bbuf, int high, int low) { + if (low < 0xDC00 && low > 0xDFFF) { + bbuf.put(REPL); + return false; } + int c = 0x10000 + ((high & 0x7FF) << 10) + (low & 0x3FF); + c = 0x808080F0 | (c >> 18) | ((c >> 4) & 0x00003F00) | ((c << 10) & 0x003F0000) | (c << 24 & 0x3F000000); + bbuf.putInt(c); + return true; } } diff --git a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java index bff41d55..9667389d 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java +++ b/unknow-server-util/src/main/java/unknow/server/util/io/ByteBufferReader.java @@ -26,23 +26,27 @@ private void decode() throws IOException { if (buf.hasRemaining() || in.isOef()) return; buf.compact(); - decoder.decode(in.buffer(), buf, in.hasRemaining() && in.isClosed()); + boolean end = in.hasRemaining() && in.isClosed(); + if (!end || in.buffer().hasRemaining()) + decoder.decode(in.buffer(), buf, end); + else + decoder.flush(buf); buf.flip(); } @Override public int read() throws IOException { - if (in.isOef()) - return -1; decode(); + if (!buf.hasRemaining()) + return -1; return buf.get(); } @Override public int read(char[] cbuf, int off, int len) throws IOException { - if (in.isOef()) - return -1; decode(); + if (!buf.hasRemaining()) + return -1; len = Math.min(len, buf.remaining()); buf.get(cbuf, off, len); return len; @@ -50,9 +54,9 @@ public int read(char[] cbuf, int off, int len) throws IOException { @Override public int read(CharBuffer target) throws IOException { - if (in.isOef()) - return -1; decode(); + if (!buf.hasRemaining()) + return -1; int l = buf.remaining(); int r = target.remaining(); if (r > l) { @@ -67,9 +71,9 @@ public int read(CharBuffer target) throws IOException { @Override public long skip(long n) throws IOException { - if (in.isOef()) - return 0; decode(); + if (!buf.hasRemaining()) + return 0; int l = (int) Math.min(n, buf.remaining()); buf.position(buf.position() + l); return n; diff --git a/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java new file mode 100644 index 00000000..c15641d4 --- /dev/null +++ b/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java @@ -0,0 +1,132 @@ +package unknow.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +public class DecoderTest { + static Stream utf8Strings() { + return Stream.of("", "hello", "éàç", "こんにちは", "😀😃😄😁", "hello é 😀", "𐍈"); + } + + static Stream invalidUtf8() { + return Stream.of(new byte[] { (byte) 0x80 }, new byte[] { (byte) 0xC0 }, new byte[] { (byte) 0xE0, (byte) 0x80 }, new byte[] { (byte) 0xF0, (byte) 0x80, (byte) 0x80 }, + new byte[] { (byte) 0xC0, (byte) 0xAF }, new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, new byte[] { (byte) 0xF8 }, new byte[] { (byte) 0xFF }); + } + + protected Decoder decoder() { + return new Decoder.DefaultDecoder(StandardCharsets.UTF_8.newDecoder()); + } + + protected void decodeAll(Decoder decoder, ByteBuffer bbuf, CharBuffer cbuf) { + decoder.decode(bbuf, cbuf, true); + while (decoder.flush(cbuf)) + ; + } + + protected void decodeChunck(Decoder decoder, byte[] bytes, CharBuffer cbuf) { + for (int i = 0; i < bytes.length; i++) { + ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { bytes[i] }); + decoder.decode(bbuf, cbuf, i == bytes.length - 1); + } + while (decoder.flush(cbuf)) + ; + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPath(String input) { + Decoder decoder = decoder(); + ByteBuffer bbuf = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)); + CharBuffer cbuf = CharBuffer.allocate(100); + decodeAll(decoder, bbuf, cbuf); + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPathChuncked(String input) { + Decoder decoder = decoder(); + + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + CharBuffer cbuf = CharBuffer.allocate(100); + decodeChunck(decoder, bytes, cbuf); + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testFastPathError(byte[] input) { + Decoder decoder = decoder(); + + ByteBuffer bbuf = ByteBuffer.wrap(input); + CharBuffer cbuf = CharBuffer.allocate(100); + + assertThrows(IllegalArgumentException.class, () -> decodeAll(decoder, bbuf, cbuf)); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testFastPathErrorChuncked(byte[] bytes) { + Decoder decoder = decoder(); + + CharBuffer cbuf = CharBuffer.allocate(100); + + assertThrows(IllegalArgumentException.class, () -> decodeChunck(decoder, bytes, cbuf)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPath(String input) { + Decoder decoder = decoder(); + + ByteBuffer bbuf = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)).asReadOnlyBuffer(); + CharBuffer cbuf = CharBuffer.allocate(100); + + decodeAll(decoder, bbuf, cbuf); + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPathChuncked(String input) { + Decoder decoder = decoder(); + + byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + CharBuffer cbuf = CharBuffer.allocate(100); + + decodeChunck(decoder, bytes, cbuf); + + assertEquals(input, cbuf.flip().toString()); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testSlowPathError(byte[] input) { + Decoder decoder = decoder(); + + ByteBuffer bbuf = ByteBuffer.wrap(input).asReadOnlyBuffer(); + CharBuffer cbuf = CharBuffer.allocate(100); + + assertThrows(IllegalArgumentException.class, () -> decodeAll(decoder, bbuf, cbuf)); + } + + @ParameterizedTest + @MethodSource("invalidUtf8") + void testSlowPathErrorChuncked(byte[] bytes) { + Decoder decoder = decoder(); + + CharBuffer cbuf = CharBuffer.allocate(100); + + assertThrows(IllegalArgumentException.class, () -> decodeChunck(decoder, bytes, cbuf)); + } +} diff --git a/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java new file mode 100644 index 00000000..0fa8c8a3 --- /dev/null +++ b/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java @@ -0,0 +1,85 @@ +package unknow.server.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class EncoderTest { + static Stream utf8Strings() { + return Stream.of(Arguments.of("", ""), Arguments.of("hello", "hello"), Arguments.of("éàç", "éàç"), Arguments.of("こんにちは", "こんにちは"), + Arguments.of("😀😃😄😁", "😀😃😄😁"), Arguments.of("hello é 😀", "hello é 😀"), Arguments.of("𐍈", "𐍈"), Arguments.of("\uD800", "\uFFFD"), + Arguments.of("\uDC00", "\uFFFD"), Arguments.of("\uD800A", "\uFFFDA"), Arguments.of("A\uDC00", "A\uFFFD")); + } + + protected Encoder encoder() { + return new Encoder.DefaultEncoder(StandardCharsets.UTF_8.newEncoder().replaceWith(Encoder.REPL).onMalformedInput(CodingErrorAction.REPLACE)); + } + + private void encodeAll(Encoder encoder, CharBuffer cbuf, ByteBuffer bbuf) { + encoder.encode(cbuf, bbuf, true); + while (encoder.flush(bbuf)) + ; + } + + private void encodeChunck(Encoder encoder, String input, ByteBuffer bbuf) { + char[] chars = input.toCharArray(); + for (int i = 0; i < chars.length; i++) { + CharBuffer cbuf = CharBuffer.wrap(new char[] { chars[i] }); + encoder.encode(cbuf, bbuf, i == chars.length - 1); + } + while (encoder.flush(bbuf)) + ; + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPath(String input, String expected) { + Encoder encoder = encoder(); + CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + encodeAll(encoder, cbuf, bbuf); + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testFastPathChuncked(String input, String expected) { + Encoder encoder = encoder(); + ByteBuffer bbuf = ByteBuffer.allocate(100); + encodeChunck(encoder, input, bbuf); + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPath(String input, String expected) { + Encoder encoder = encoder(); + CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + encodeAll(encoder, cbuf, bbuf); + + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @MethodSource("utf8Strings") + void testSlowPathChuncked(String input, String expected) { + Encoder encoder = encoder(); + ByteBuffer bbuf = ByteBuffer.allocate(100); + + encodeChunck(encoder, input, bbuf); + + assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + } + +} diff --git a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java index a966366c..badff088 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/Utf8DecoderTest.java @@ -1,133 +1,8 @@ package unknow.server.util; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; -import java.util.stream.Stream; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -public class Utf8DecoderTest { - static Stream utf8Strings() { - return Stream.of("", "hello", "éàç", "こんにちは", "😀😃😄😁", "hello é 😀", "𐍈"); - } - - static Stream invalidUtf8() { - return Stream.of(new byte[] { (byte) 0x80 }, new byte[] { (byte) 0xC0 }, new byte[] { (byte) 0xE0, (byte) 0x80 }, - new byte[] { (byte) 0xF0, (byte) 0x80, (byte) 0x80 }, - new byte[] { (byte) 0xC0, (byte) 0xAF }, - new byte[] { (byte) 0xE0, (byte) 0x9F, (byte) 0x80 }, - new byte[] { (byte) 0xF8 }, new byte[] { (byte) 0xFF }); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testFastPath(String input) { - Decoder decoder = new Utf8Decoder(); - ByteBuffer bbuf = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)); - CharBuffer cbuf = CharBuffer.allocate(100); - - decoder.decode(bbuf, cbuf, true); - - assertEquals(input, cbuf.flip().toString()); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testFastPathChuncked(String input) { - Decoder decoder = new Utf8Decoder(); - - byte[] bytes = input.getBytes(StandardCharsets.UTF_8); - CharBuffer cbuf = CharBuffer.allocate(100); - - for (int i = 0; i < bytes.length; i++) { - ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { bytes[i] }); - decoder.decode(bbuf, cbuf, i == bytes.length - 1); - } - - assertEquals(input, cbuf.flip().toString()); - } - - @ParameterizedTest - @MethodSource("invalidUtf8") - void testFastPathError(byte[] input) { - Decoder decoder = new Utf8Decoder(); - - ByteBuffer bbuf = ByteBuffer.wrap(input); - CharBuffer cbuf = CharBuffer.allocate(10); - - assertThrows(IllegalArgumentException.class, () -> decoder.decode(bbuf, cbuf, true)); - } - - @ParameterizedTest - @MethodSource("invalidUtf8") - void testFastPathErrorChuncked(byte[] input) { - Decoder decoder = Decoder.from(StandardCharsets.UTF_8); - - CharBuffer cbuf = CharBuffer.allocate(10); - - assertThrows(IllegalArgumentException.class, () -> { - for (int i = 0; i < input.length; i++) { - ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { input[i] }); - decoder.decode(bbuf, cbuf, i == input.length - 1); - } - }); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testSlowPath(String input) { - ByteBuffer bbuf = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)).asReadOnlyBuffer(); - CharBuffer cbuf = CharBuffer.allocate(100); - - new Utf8Decoder().decode(bbuf, cbuf, true); - - assertEquals(input, cbuf.flip().toString()); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testSlowPathChuncked(String input) { - Decoder decoder = new Utf8Decoder(); - - byte[] bytes = input.getBytes(StandardCharsets.UTF_8); - CharBuffer cbuf = CharBuffer.allocate(100); - - for (int i = 0; i < bytes.length; i++) { - ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { bytes[i] }).asReadOnlyBuffer(); - decoder.decode(bbuf, cbuf, i == bytes.length - 1); - } - - assertEquals(input, cbuf.flip().toString()); - } - - @ParameterizedTest - @MethodSource("invalidUtf8") - void testSlowPathError(byte[] input) { - Decoder decoder = new Utf8Decoder(); - - ByteBuffer bbuf = ByteBuffer.wrap(input).asReadOnlyBuffer(); - CharBuffer cbuf = CharBuffer.allocate(10); - - assertThrows(IllegalArgumentException.class, () -> decoder.decode(bbuf, cbuf, true)); - } - - @ParameterizedTest - @MethodSource("invalidUtf8") - void testSlowPathErrorChuncked(byte[] input) { - Decoder decoder = Decoder.from(StandardCharsets.UTF_8); - - CharBuffer cbuf = CharBuffer.allocate(10); - - assertThrows(IllegalArgumentException.class, () -> { - for (int i = 0; i < input.length; i++) { - ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { input[i] }).asReadOnlyBuffer(); - decoder.decode(bbuf, cbuf, i == input.length - 1); - } - }); +public class Utf8DecoderTest extends DecoderTest { + @Override + protected unknow.server.util.Decoder decoder() { + return new Utf8Decoder(); } } diff --git a/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java index 2be8e468..ddaaea91 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/Utf8EncoderTest.java @@ -1,75 +1,10 @@ package unknow.server.util; -import static org.junit.jupiter.api.Assertions.assertEquals; +public class Utf8EncoderTest extends EncoderTest { -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.StandardCharsets; -import java.util.stream.Stream; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -public class Utf8EncoderTest { - static Stream utf8Strings() { - return Stream.of(Arguments.of("", ""), Arguments.of("hello", "hello"), Arguments.of("éàç", "éàç"), Arguments.of("こんにちは", "こんにちは"), - Arguments.of("😀😃😄😁", "😀😃😄😁"), Arguments.of("hello é 😀", "hello é 😀"), Arguments.of("𐍈", "𐍈"), Arguments.of("\uD800", "\uFFFD"), - Arguments.of("\uDC00", "\uFFFD"), Arguments.of("\uD800A", "\uFFFDA"), Arguments.of("A\uDC00", "A\uFFFD")); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testFastPath(String input, String expected) { - Encoder decoder = new Utf8Encoder(); - CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); - ByteBuffer bbuf = ByteBuffer.allocate(100); - - decoder.encode(cbuf, bbuf, true); - - assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testFastPathChuncked(String input, String expected) { - Encoder decoder = new Utf8Encoder(); - char[] chars = input.toCharArray(); - ByteBuffer bbuf = ByteBuffer.allocate(100); - - for (int i = 0; i < chars.length; i++) { - CharBuffer cbuf = CharBuffer.wrap(new char[] { chars[i] }); - decoder.encode(cbuf, bbuf, i == chars.length - 1); - } - - assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testSlowPath(String input, String expected) { - Encoder encoder = new Utf8Encoder(); - CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); - ByteBuffer bbuf = ByteBuffer.allocate(100); - - encoder.encode(cbuf, bbuf, true); - - assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); - } - - @ParameterizedTest - @MethodSource("utf8Strings") - void testSlowPathChuncked(String input, String expected) { - Encoder decoder = new Utf8Encoder(); - char[] chars = input.toCharArray(); - ByteBuffer bbuf = ByteBuffer.allocate(100); - - for (int i = 0; i < chars.length; i++) { - CharBuffer cbuf = CharBuffer.wrap(new char[] { chars[i] }).asReadOnlyBuffer(); - decoder.encode(cbuf, bbuf, i == chars.length - 1); - } - - assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); + @Override + protected unknow.server.util.Encoder encoder() { + return new Utf8Encoder(); } } From 5c59b4cbb60fe74d7c6a4a95b43ce3ee3efe7671 Mon Sep 17 00:00:00 2001 From: Unknow Date: Sun, 12 Apr 2026 16:13:17 +0200 Subject: [PATCH 26/31] use ascii charsetencoder/decoder to speedup --- .../java/unknow/server/util/Utf8Decoder.java | 51 ++++------------ .../java/unknow/server/util/Utf8Encoder.java | 59 ++++++++++--------- 2 files changed, 40 insertions(+), 70 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index 3db045a9..f7184b46 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -5,12 +5,15 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.StandardCharsets; public class Utf8Decoder implements Decoder { - private static final VarHandle LONG = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN); private static final VarHandle INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); private static final VarHandle SHORT = MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.LITTLE_ENDIAN); + private final CharsetDecoder ascii = StandardCharsets.US_ASCII.newDecoder(); + /** code point in building */ private int cp; /** min code point allowed */ @@ -35,39 +38,24 @@ public boolean flush(CharBuffer cbuf) { } private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { + byte[] barr = bbuf.array(); - int bpos = bbuf.position() + bbuf.arrayOffset(); int blim = bbuf.limit() + bbuf.arrayOffset(); char[] carr = cbuf.array(); - int cpos = cbuf.position() + cbuf.arrayOffset(); int clim = cbuf.limit() - 1 + cbuf.arrayOffset(); if (r > 0) { + int bpos = bbuf.position() + bbuf.arrayOffset(); + int cpos = cbuf.position() + cbuf.arrayOffset(); remainingArray(bbuf, bpos, cbuf, cpos, cp, r); if (r > 0) return; - bpos = bbuf.position() + bbuf.arrayOffset(); - cpos = cbuf.position() + cbuf.arrayOffset(); - } - int max = Math.min(clim, cpos + blim - bpos) - 8; - while (cpos < max) { - long l = (long) LONG.get(barr, bpos); - if ((l & 0x8080808080808080L) != 0L) - break; - carr[cpos] = (char) l; - carr[cpos + 1] = (char) (l >>> 8); - carr[cpos + 2] = (char) (l >>> 16); - carr[cpos + 3] = (char) (l >>> 24); - carr[cpos + 4] = (char) (l >>> 32); - carr[cpos + 5] = (char) (l >>> 40); - carr[cpos + 6] = (char) (l >>> 48); - carr[cpos + 7] = (char) (l >>> 56); - bpos += 8; - cpos += 8; } + ascii.decode(bbuf, cbuf, false); + int bpos = bbuf.position() + bbuf.arrayOffset(); + int cpos = cbuf.position() + cbuf.arrayOffset(); int cp; while (bpos < blim && cpos < clim) { int b = barr[bpos++]; - if (b >= 0) carr[cpos++] = (char) b; else if (b < -64) @@ -170,29 +158,10 @@ private static void updatePos(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cp cbuf.position(cpos - cbuf.arrayOffset()); } - private final char[] CARR = new char[8]; - private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { if (r > 0 && slowRemaining(bbuf, cbuf, cp, r)) return; - int max = Math.min(bbuf.limit(), bbuf.position() + cbuf.remaining()) - 8; - int bpos = bbuf.position(); - while (bpos < max) { - long l = bbuf.getLong(bpos); - if ((l & 0x8080808080808080L) != 0L) - break; - CARR[0] = (char) l; - CARR[1] = (char) (l >>> 8); - CARR[2] = (char) (l >>> 16); - CARR[3] = (char) (l >>> 24); - CARR[4] = (char) (l >>> 32); - CARR[5] = (char) (l >>> 40); - CARR[6] = (char) (l >>> 48); - CARR[7] = (char) (l >>> 56); - cbuf.put(CARR); - bpos += 8; - } int cp; while (bbuf.hasRemaining() && cbuf.remaining() > 1) { int b = bbuf.get(); diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index 55f5d38d..a6f8d07c 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -5,13 +5,17 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.StandardCharsets; public class Utf8Encoder implements Encoder { private static final VarHandle INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); private static final int repl = 0x00BDBFEF; - private int cp; + private final CharsetEncoder ascii = StandardCharsets.US_ASCII.newEncoder(); + + private int hi; @Override public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { @@ -25,50 +29,47 @@ public void encode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { @Override public boolean flush(ByteBuffer bbuf) { - if (cp != 0) { + if (hi != 0) { if (bbuf.remaining() < 3) return true; bbuf.put(REPL); - cp = 0; + hi = 0; } return false; } private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { char[] carr = cbuf.array(); - int cpos = cbuf.position() + cbuf.arrayOffset(); - int clim = cbuf.limit() + cbuf.arrayOffset(); + int coff = cbuf.arrayOffset(); + int clim = cbuf.limit() + coff; byte[] barr = bbuf.array(); - int bpos = bbuf.position() + bbuf.arrayOffset(); - int blim = bbuf.limit() - 3 + bbuf.arrayOffset(); + int boff = bbuf.arrayOffset(); + int blim = bbuf.limit() - 3 + boff; - if (cp != 0) { + if (hi != 0) { + int bpos = bbuf.position() + boff; + int cpos = cbuf.position() + coff; int low = carr[cpos]; if (low >= 0xDC00 && low <= 0xDFFF) { - cpos++; - int c = 0x10000 + ((cp & 0x7FF) << 10) + (low & 0x3FF); + cbuf.position(cpos - coff + 1); + int c = 0x10000 + ((hi & 0x7FF) << 10) + (low & 0x3FF); c = 0x808080F0 | (c >> 18) | ((c >> 4) & 0x00003F00) | ((c << 10) & 0x003F0000) | (c << 24 & 0x3F000000); INT.set(barr, bpos, c); - bpos += 4; + bbuf.position(bpos - boff + 4); } else { INT.set(barr, bpos, repl); - bpos += 3; + bbuf.position(bpos - boff + 3); } - cp = 0; + hi = 0; } - int max = Math.min(clim, cpos + blim - bpos); - int start = cpos; - while (cpos < max && carr[cpos] < 0x80) - cpos++; - int len = cpos - start; - for (int i = 0; i < len; i++) - barr[bpos + i] = (byte) carr[start + i]; - bpos += len; + ascii.encode(cbuf, bbuf, false); + int bpos = bbuf.position() + boff; + int cpos = cbuf.position() + coff; while (cpos < clim && bpos < blim) { - int code = carr[cpos++]; + char code = carr[cpos++]; if (code < 0x80) { barr[bpos++] = (byte) code; } else if (code < 0x800) { @@ -97,7 +98,7 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { INT.set(barr, bpos, repl); bpos += 3; } else { - cp = code; + hi = code; break; } } else if (code < 0xE000) { // lone low surrogate @@ -109,17 +110,17 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { bpos += 3; } } - cbuf.position(cpos - cbuf.arrayOffset()); - bbuf.position(bpos - bbuf.arrayOffset()); + cbuf.position(cpos - coff); + bbuf.position(bpos - boff); } private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { - if (cp != 0) { + if (hi != 0) { int cpos = cbuf.position(); int low = cbuf.get(cpos); - if (slowSurrogate(bbuf, cp, low)) + if (slowSurrogate(bbuf, hi, low)) cbuf.position(cpos + 1); - cp = 0; + hi = 0; } while (cbuf.hasRemaining() && bbuf.remaining() >= 4) { int code = cbuf.get(); @@ -141,7 +142,7 @@ private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { } else if (endOfInput) bbuf.put(REPL); else { - cp = code; + hi = code; break; } } else if (code < 0xE000) // lone low surrogate From 5d5fb1316cd764aac83df9d693e17a737ad8c281 Mon Sep 17 00:00:00 2001 From: Unknow Date: Sun, 12 Apr 2026 18:10:33 +0200 Subject: [PATCH 27/31] update from sonar --- .../unknow/server/bench/EncoderDecoder.java | 14 +--- .../java/unknow/server/nio/NIOConnection.java | 32 ++------ .../java/unknow/server/nio/NIOSSLHandler.java | 12 +-- .../java/unknow/server/util/Utf8Decoder.java | 80 +++++++++---------- .../java/unknow/server/util/Utf8Encoder.java | 10 +-- .../java/unknow/server/util/DecoderTest.java | 19 +++-- .../java/unknow/server/util/EncoderTest.java | 16 ++-- 7 files changed, 73 insertions(+), 110 deletions(-) diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index f600eb87..c3ed9270 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -123,6 +123,7 @@ public void setup() { case "complex": init(COMPLEX); break; + default: } } @@ -133,19 +134,6 @@ private void init(String str) { } } -// @Benchmark -// public ByteBuffer encoderUtf8Servlet(EncodeData data) { -// ByteBuffer b = ByteBuffer.allocate(4096); -// Utf8Encoder.encode(data.str, b); -// return b.flip(); -// } -// -// @Benchmark -// public String decoderUtf8Servlet(DecodeData data) { -// ByteBuffer b = ByteBuffer.wrap(data.bytes); -// return new Utf8Decoder().append(b.array(), 0, b.limit()).done(); -// } - @Benchmark public ByteBuffer encoderUtf8(Data data) { ByteBuffer b = ByteBuffer.allocate(4096); diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index 075a0167..36788595 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -100,7 +100,7 @@ public void init(NIOConnection co, long now, SSLEngine sslEngine) throws IOExcep * @throws InterruptedException in case of interruption * @throws IOException */ - public final void write(ByteBuffer buf) throws InterruptedException, IOException { + public final void write(ByteBuffer buf) throws IOException { if (!key.isValid()) throw new IOException("already closed"); pending.offer(buf); @@ -108,12 +108,6 @@ public final void write(ByteBuffer buf) throws InterruptedException, IOException execute(writeCheck); } - public final void flush() { -// if (!hasPendingWrites()) -// return; -// key.selector().wakeup(); - } - /** * @return the current outputStream */ @@ -258,12 +252,7 @@ public synchronized void write(byte[] b, int off, int len) throws IOException { if (len < BUF_SIZE) buf.put(b, off, len); else - try { - h.write(ByteBuffer.wrap(Arrays.copyOfRange(b, off, off + len))); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException(e); - } + h.write(ByteBuffer.wrap(Arrays.copyOfRange(b, off, off + len))); } else if (len == r) { buf.put(b, off, len); writeBuffer(); @@ -281,12 +270,7 @@ public synchronized void write(ByteBuffer b) throws IOException { if (h == null) throw new IOException("already closed"); writeBuffer(); - try { - h.write(b); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException(e); - } + h.write(b); } @Override @@ -311,19 +295,13 @@ public synchronized void flush() throws IOException { if (h == null) return; writeBuffer(); - h.flush(); } private void writeBuffer() throws IOException { if (h == null || buf.position() == 0) return; - try { - h.write(buf.flip()); - buf = ByteBuffer.allocate(BUF_SIZE); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException(e); - } + h.write(buf.flip()); + buf = ByteBuffer.allocate(BUF_SIZE); } } diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java index 35d5e4c5..672cad8e 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOSSLHandler.java @@ -10,7 +10,6 @@ import javax.net.ssl.SSLEngineResult; import javax.net.ssl.SSLEngineResult.HandshakeStatus; import javax.net.ssl.SSLEngineResult.Status; -import javax.net.ssl.SSLException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,8 +146,6 @@ public boolean finishClosing(long now) { try { co.write(EMPTY); } catch (@SuppressWarnings("unused") IOException e) { // ok - } catch (@SuppressWarnings("unused") InterruptedException e) { - Thread.currentThread().interrupt(); } return false; } @@ -172,13 +169,8 @@ private boolean checkHandshake(HandshakeStatus hs, long now) throws IOException hs = r.getHandshakeStatus(); break; case NEED_WRAP: - try { - co.write(EMPTY); - return true; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SSLException("Handshake interrupted", e); - } + co.write(EMPTY); + return true; case FINISHED: onHandshakeDone(sslEngine, now); // fallthrough diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java index f7184b46..1c48cdd1 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Decoder.java @@ -53,7 +53,7 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { ascii.decode(bbuf, cbuf, false); int bpos = bbuf.position() + bbuf.arrayOffset(); int cpos = cbuf.position() + cbuf.arrayOffset(); - int cp; + int code; while (bpos < blim && cpos < clim) { int b = barr[bpos++]; if (b >= 0) @@ -61,9 +61,9 @@ private void fastDecode(ByteBuffer bbuf, CharBuffer cbuf) { else if (b < -64) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 start byte: 0x" + Integer.toString(b & 0xFF, 16)); else if (b < -32) { - cp = b & 0x1F; + code = b & 0x1F; if (bpos == blim) { - this.cp = cp; + this.cp = code; r = 1; minCp = 0x80; updatePos(bbuf, bpos, cbuf, cpos); @@ -72,45 +72,45 @@ else if (b < -32) { b = barr[bpos++]; if ((b & 0xc0) != 0x80) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); - cp = (cp << 6) ^ (b & 0x3F); - if (cp < 0x80) + code = (code << 6) | (b & 0x3F); + if (code < 0x80) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence"); - carr[cpos++] = (char) cp; + carr[cpos++] = (char) code; } else if (b < -16) { - cp = b & 0x0F; + code = b & 0x0F; if (bpos + 1 >= blim) { minCp = 0x800; - remainingArray(bbuf, bpos, cbuf, cpos, cp, 2); + remainingArray(bbuf, bpos, cbuf, cpos, code, 2); return; } short s = (short) SHORT.get(barr, bpos); if ((s & 0xc0c0) != 0x8080) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); - cp = (cp << 12) ^ ((s & 0x3F) << 6) ^ ((s >> 8) & 0x3F); - if (cp < 0x800) + code = (code << 12) | ((s & 0x3F) << 6) | ((s >> 8) & 0x3F); + if (code < 0x800) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence"); - carr[cpos++] = (char) cp; + carr[cpos++] = (char) code; bpos += 2; } else if (b < -8) { - cp = b & 0x07; + code = b & 0x07; if (bpos + 3 >= blim) { minCp = 0x10000; - remainingArray(bbuf, bpos, cbuf, cpos, cp, 3); + remainingArray(bbuf, bpos, cbuf, cpos, code, 3); return; } int i = (int) INT.get(barr, bpos); if ((i & 0x00C0C0C0) != 0x00808080) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); - cp = (cp << 18) ^ ((i & 0x3F) << 12) ^ (((i >> 8) & 0x3F) << 6) ^ (((i >> 16) & 0x3F)); - if (cp < 0x10000) + code = (code << 18) | ((i & 0x3F) << 12) | (((i >> 8) & 0x3F) << 6) | ((i >> 16) & 0x3F); + if (code < 0x10000) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence"); - if (cp > 0xFFFF) { // Surrogate pair - cp -= 0x10000; - carr[cpos++] = (char) ((cp >> 10) ^ 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) ^ 0xDC00); + if (code > 0xFFFF) { // Surrogate pair + code -= 0x10000; + carr[cpos++] = (char) ((code >> 10) | 0xD800); + carr[cpos++] = (char) ((code & 0x3FF) | 0xDC00); } else - carr[cpos++] = (char) cp; + carr[cpos++] = (char) code; bpos += 3; } else error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); @@ -128,14 +128,14 @@ private void remainingArray(ByteBuffer bbuf, int bpos, CharBuffer cbuf, int cpos int b = barr[bpos++]; if ((b & 0xc0) != 0x80) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 continuation byte"); - cp = (cp << 6) ^ (b & 0x3F); + cp = (cp << 6) | (b & 0x3F); if (--r == 0) { if (cp < minCp) error(bbuf, bpos, cbuf, cpos, "Invalid UTF-8 overlong sequence " + Integer.toString(cp, 16) + " < " + Integer.toString(minCp, 16)); if (cp > 0xFFFF) { // Surrogate pair cp -= 0x10000; - carr[cpos++] = (char) ((cp >> 10) ^ 0xD800); - carr[cpos++] = (char) ((cp & 0x3FF) ^ 0xDC00); + carr[cpos++] = (char) ((cp >> 10) | 0xD800); + carr[cpos++] = (char) ((cp & 0x3FF) | 0xDC00); } else carr[cpos++] = (char) cp; this.r = r; @@ -162,7 +162,7 @@ private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { if (r > 0 && slowRemaining(bbuf, cbuf, cp, r)) return; - int cp; + int code; while (bbuf.hasRemaining() && cbuf.remaining() > 1) { int b = bbuf.get(); if (b >= 0) @@ -170,50 +170,50 @@ private void slowDecode(ByteBuffer bbuf, CharBuffer cbuf) { else if (b < -64) throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b & 0xFF, 16)); else if (b < -32) { - cp = b & 0x1F; + code = b & 0x1F; if (!bbuf.hasRemaining()) { minCp = 0x80; r = 1; - this.cp = cp; + this.cp = code; return; } b = bbuf.get(); if ((b & 0xc0) != 0x80) throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); - cp = (cp << 6) ^ (b & 0x3F); - if (cp < 0x80) + code = (code << 6) | (b & 0x3F); + if (code < 0x80) throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - cbuf.put((char) cp); + cbuf.put((char) code); } else if (b < -16) { - cp = b & 0x0F; + code = b & 0x0F; if (bbuf.remaining() < 2) { minCp = 0x800; - slowRemaining(bbuf, cbuf, cp, 2); + slowRemaining(bbuf, cbuf, code, 2); return; } short s = bbuf.getShort(); if ((s & 0xc0c0) != 0x8080) throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); - cp = (cp << 12) ^ ((s & 0x3F) << 6) ^ ((s >> 8) & 0x3F); - if (cp < 0x800) + code = (code << 12) | ((s & 0x3F) << 6) | ((s >> 8) & 0x3F); + if (code < 0x800) throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - cbuf.put((char) cp); + cbuf.put((char) code); } else if (b < -8) { - cp = b & 0x07; + code = b & 0x07; if (bbuf.remaining() < 4) { minCp = 0x10000; - slowRemaining(bbuf, cbuf, cp, 3); + slowRemaining(bbuf, cbuf, code, 3); return; } int i = bbuf.getInt(bbuf.position()); bbuf.position(bbuf.position() + 3); if ((i & 0x00C0C0C0) != 0x00808080) throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); - cp = (cp << 18) ^ ((i & 0x3F) << 12) ^ (((i >> 8) & 0x3F) << 6) ^ (((i >> 16) & 0x3F)); - if (cp < 0x10000) + code = (code << 18) | ((i & 0x3F) << 12) | (((i >> 8) & 0x3F) << 6) | ((i >> 16) & 0x3F); + if (code < 0x10000) throw new IllegalArgumentException("Invalid UTF-8 overlong sequence"); - slowAppend(cbuf, cp); + slowAppend(cbuf, code); } else throw new IllegalArgumentException("Invalid UTF-8 start byte: 0x" + Integer.toString(b, 16)); } @@ -224,7 +224,7 @@ private boolean slowRemaining(ByteBuffer bbuf, CharBuffer cbuf, int cp, int r) { int b = bbuf.get(); if ((b & 0xc0) != 0x80) throw new IllegalArgumentException("Invalid UTF-8 continuation byte"); - cp = (cp << 6) ^ (b & 0x3F); + cp = (cp << 6) | (b & 0x3F); if (--r == 0) slowAppend(cbuf, cp); } diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index a6f8d07c..4ec5ccf4 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -11,7 +11,7 @@ public class Utf8Encoder implements Encoder { private static final VarHandle INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); - private static final int repl = 0x00BDBFEF; + private static final int INT_REPL = 0x00BDBFEF; private final CharsetEncoder ascii = StandardCharsets.US_ASCII.newEncoder(); @@ -58,7 +58,7 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { INT.set(barr, bpos, c); bbuf.position(bpos - boff + 4); } else { - INT.set(barr, bpos, repl); + INT.set(barr, bpos, INT_REPL); bbuf.position(bpos - boff + 3); } hi = 0; @@ -91,18 +91,18 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { INT.set(barr, bpos, c); bpos += 4; } else { - INT.set(barr, bpos, repl); + INT.set(barr, bpos, INT_REPL); bpos += 3; } } else if (endOfInput) { - INT.set(barr, bpos, repl); + INT.set(barr, bpos, INT_REPL); bpos += 3; } else { hi = code; break; } } else if (code < 0xE000) { // lone low surrogate - INT.set(barr, bpos, repl); + INT.set(barr, bpos, INT_REPL); bpos += 3; } else { int c = 0x008080E0 | (code >> 12) | ((code << 2) & 0x3F00) | (code << 16 & 0x3F0000); diff --git a/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java index c15641d4..5a728b3c 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java @@ -31,10 +31,11 @@ protected void decodeAll(Decoder decoder, ByteBuffer bbuf, CharBuffer cbuf) { ; } - protected void decodeChunck(Decoder decoder, byte[] bytes, CharBuffer cbuf) { - for (int i = 0; i < bytes.length; i++) { - ByteBuffer bbuf = ByteBuffer.wrap(new byte[] { bytes[i] }); - decoder.decode(bbuf, cbuf, i == bytes.length - 1); + protected void decodeChunck(Decoder decoder, ByteBuffer input, CharBuffer cbuf) { + int l = input.limit(); + for (int i = 0; i < l; i++) { + ByteBuffer bbuf = input.slice(i, 1); + decoder.decode(bbuf, cbuf, i == l - 1); } while (decoder.flush(cbuf)) ; @@ -56,7 +57,7 @@ void testFastPath(String input) { void testFastPathChuncked(String input) { Decoder decoder = decoder(); - byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + ByteBuffer bytes = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)); CharBuffer cbuf = CharBuffer.allocate(100); decodeChunck(decoder, bytes, cbuf); assertEquals(input, cbuf.flip().toString()); @@ -78,9 +79,10 @@ void testFastPathError(byte[] input) { void testFastPathErrorChuncked(byte[] bytes) { Decoder decoder = decoder(); + ByteBuffer bbuf = ByteBuffer.wrap(bytes); CharBuffer cbuf = CharBuffer.allocate(100); - assertThrows(IllegalArgumentException.class, () -> decodeChunck(decoder, bytes, cbuf)); + assertThrows(IllegalArgumentException.class, () -> decodeChunck(decoder, bbuf, cbuf)); } @ParameterizedTest @@ -101,7 +103,7 @@ void testSlowPath(String input) { void testSlowPathChuncked(String input) { Decoder decoder = decoder(); - byte[] bytes = input.getBytes(StandardCharsets.UTF_8); + ByteBuffer bytes = ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)).asReadOnlyBuffer(); CharBuffer cbuf = CharBuffer.allocate(100); decodeChunck(decoder, bytes, cbuf); @@ -125,8 +127,9 @@ void testSlowPathError(byte[] input) { void testSlowPathErrorChuncked(byte[] bytes) { Decoder decoder = decoder(); + ByteBuffer bbuf = ByteBuffer.wrap(bytes).asReadOnlyBuffer(); CharBuffer cbuf = CharBuffer.allocate(100); - assertThrows(IllegalArgumentException.class, () -> decodeChunck(decoder, bytes, cbuf)); + assertThrows(IllegalArgumentException.class, () -> decodeChunck(decoder, bbuf, cbuf)); } } diff --git a/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java index 0fa8c8a3..7b1a4463 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java @@ -29,11 +29,11 @@ private void encodeAll(Encoder encoder, CharBuffer cbuf, ByteBuffer bbuf) { ; } - private void encodeChunck(Encoder encoder, String input, ByteBuffer bbuf) { - char[] chars = input.toCharArray(); - for (int i = 0; i < chars.length; i++) { - CharBuffer cbuf = CharBuffer.wrap(new char[] { chars[i] }); - encoder.encode(cbuf, bbuf, i == chars.length - 1); + private void encodeChunck(Encoder encoder, CharBuffer input, ByteBuffer bbuf) { + int l = input.limit(); + for (int i = 0; i < l; i++) { + CharBuffer cbuf = input.slice(i, 1); + encoder.encode(cbuf, bbuf, i == l - 1); } while (encoder.flush(bbuf)) ; @@ -54,8 +54,9 @@ void testFastPath(String input, String expected) { @MethodSource("utf8Strings") void testFastPathChuncked(String input, String expected) { Encoder encoder = encoder(); + CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); ByteBuffer bbuf = ByteBuffer.allocate(100); - encodeChunck(encoder, input, bbuf); + encodeChunck(encoder, cbuf, bbuf); assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); } @@ -75,9 +76,10 @@ void testSlowPath(String input, String expected) { @MethodSource("utf8Strings") void testSlowPathChuncked(String input, String expected) { Encoder encoder = encoder(); + CharBuffer cbuf = CharBuffer.wrap(input); ByteBuffer bbuf = ByteBuffer.allocate(100); - encodeChunck(encoder, input, bbuf); + encodeChunck(encoder, cbuf, bbuf); assertEquals(expected, new String(bbuf.array(), 0, bbuf.position(), StandardCharsets.UTF_8)); } From 3b26d54859e97f45f71607f0cc29471857028dea Mon Sep 17 00:00:00 2001 From: Unknow Date: Mon, 13 Apr 2026 00:02:34 +0200 Subject: [PATCH 28/31] update --- .../java/unknow/server/util/Utf8Encoder.java | 22 ++++++++++--------- .../java/unknow/server/util/DecoderTest.java | 6 ++--- .../java/unknow/server/util/EncoderTest.java | 8 +++---- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java index 4ec5ccf4..92c7dd47 100644 --- a/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java +++ b/unknow-server-util/src/main/java/unknow/server/util/Utf8Encoder.java @@ -94,11 +94,12 @@ private void fastEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { INT.set(barr, bpos, INT_REPL); bpos += 3; } - } else if (endOfInput) { - INT.set(barr, bpos, INT_REPL); - bpos += 3; } else { - hi = code; + if (endOfInput) { + INT.set(barr, bpos, INT_REPL); + bpos += 3; + } else + hi = code; break; } } else if (code < 0xE000) { // lone low surrogate @@ -127,7 +128,7 @@ private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { if (code < 0x80) { bbuf.put((byte) code); } else if (code < 0x800) { - bbuf.putShort((short) (0x80C0 | (code >> 6) | (code & 0x3F))); + bbuf.putShort((short) (0x80C0 | (code >> 6) | ((code << 8) & 0x3F00))); } else if (code < 0xD800) { int p = bbuf.position(); int c = 0x008080E0 | (code >> 12) | ((code << 2) & 0x3F00) | (code << 16 & 0x3F0000); @@ -139,10 +140,11 @@ private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { int low = cbuf.get(cpos); if (slowSurrogate(bbuf, code, low)) cbuf.position(cpos + 1); - } else if (endOfInput) - bbuf.put(REPL); - else { - hi = code; + } else { + if (endOfInput) + bbuf.put(REPL); + else + hi = code; break; } } else if (code < 0xE000) // lone low surrogate @@ -156,7 +158,7 @@ private void slowEncode(CharBuffer cbuf, ByteBuffer bbuf, boolean endOfInput) { } private boolean slowSurrogate(ByteBuffer bbuf, int high, int low) { - if (low < 0xDC00 && low > 0xDFFF) { + if (low < 0xDC00 || low > 0xDFFF) { bbuf.put(REPL); return false; } diff --git a/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java index 5a728b3c..afe08ab6 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/DecoderTest.java @@ -33,10 +33,8 @@ protected void decodeAll(Decoder decoder, ByteBuffer bbuf, CharBuffer cbuf) { protected void decodeChunck(Decoder decoder, ByteBuffer input, CharBuffer cbuf) { int l = input.limit(); - for (int i = 0; i < l; i++) { - ByteBuffer bbuf = input.slice(i, 1); - decoder.decode(bbuf, cbuf, i == l - 1); - } + for (int i = 0; i < l; i++) + decoder.decode(input.position(i).limit(i + 1), cbuf, i == l - 1); while (decoder.flush(cbuf)) ; } diff --git a/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java b/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java index 7b1a4463..e6fc6105 100644 --- a/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java +++ b/unknow-server-util/src/test/java/unknow/server/util/EncoderTest.java @@ -31,10 +31,8 @@ private void encodeAll(Encoder encoder, CharBuffer cbuf, ByteBuffer bbuf) { private void encodeChunck(Encoder encoder, CharBuffer input, ByteBuffer bbuf) { int l = input.limit(); - for (int i = 0; i < l; i++) { - CharBuffer cbuf = input.slice(i, 1); - encoder.encode(cbuf, bbuf, i == l - 1); - } + for (int i = 0; i < l; i++) + encoder.encode(input.position(i).limit(i + 1), bbuf, i == l - 1); while (encoder.flush(bbuf)) ; } @@ -64,7 +62,7 @@ void testFastPathChuncked(String input, String expected) { @MethodSource("utf8Strings") void testSlowPath(String input, String expected) { Encoder encoder = encoder(); - CharBuffer cbuf = CharBuffer.wrap(input.toCharArray()); + CharBuffer cbuf = CharBuffer.wrap(input); ByteBuffer bbuf = ByteBuffer.allocate(100); encodeAll(encoder, cbuf, bbuf); From 4681a8fba1e6fd1ae6d8bae68c61a0c0a4d33d36 Mon Sep 17 00:00:00 2001 From: Unknow Date: Mon, 13 Apr 2026 19:20:47 +0200 Subject: [PATCH 29/31] replace some emoji with escape sequence --- .../unknow/server/bench/EncoderDecoder.java | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index c3ed9270..f6893990 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -68,41 +68,41 @@ public class EncoderDecoder { private static final String SIMPLE = "Hello world!\nÇa va bien ?\nПривет, как дела?\n你好，世界\nこんにちは世界\n👋🌍✨🔥🚀\nLorem ipsum dolor sit amet, consectetur adipiscing elit."; private static final String COMPLEX = "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" - + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" - + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" - + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" - + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" - + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" + + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" - + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" - + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" - + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" - + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" - + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" + + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" - + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" - + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" - + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" - + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" - + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" + + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" - + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" - + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" - + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" - + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" - + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" + + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" - + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" - + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" - + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" - + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" - + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" + + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" - + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 🧿👁️‍🗨️ 👁️👁️ 🌀🌀🌀\r\n" + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" - + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" - + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" - + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" - + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱"; + + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" + + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱"; @State(Scope.Thread) public static class Data { From 38388a25623a6346ad1d80334c7b039743524e96 Mon Sep 17 00:00:00 2001 From: Unknow Date: Mon, 13 Apr 2026 19:28:02 +0200 Subject: [PATCH 30/31] cleanup --- .../java/unknow/server/nio/ConnectionStats.java | 15 +++++++++++++++ .../java/unknow/server/nio/NIOConnection.java | 3 +-- .../unknow/server/nio/NIOConnectionHandler.java | 3 +++ .../src/main/java/unknow/server/nio/NIOLoop.java | 1 + .../java/unknow/server/nio/NIOServerListener.java | 1 + .../main/java/unknow/server/nio/NIOWorkers.java | 1 + .../unknow/server/servlet/HttpConnection.java | 7 +------ 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/ConnectionStats.java b/unknow-server-nio/src/main/java/unknow/server/nio/ConnectionStats.java index f9d2550f..fa2a7b6d 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/ConnectionStats.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/ConnectionStats.java @@ -1,5 +1,8 @@ package unknow.server.nio; +/** + * statistics of a nio connection + */ public class ConnectionStats { private final boolean hasPengingWrite; private final boolean closing; @@ -13,18 +16,30 @@ public ConnectionStats(boolean hasPengingWrite, boolean closing, long lastCheck, this.lastAction = lastAction; } + /** + * @return true if the connection has pending write + */ public boolean hasPengingWrite() { return hasPengingWrite; } + /** + * @return true if the connection is closing + */ public boolean isClosing() { return closing; } + /** + * @return nanotime of last connection check + */ public long lastCheck() { return lastCheck; } + /** + * @return nano time of last action on the connection + */ public long lastAction() { return lastAction; } diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java index 36788595..a1b422c0 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnection.java @@ -97,8 +97,7 @@ public void init(NIOConnection co, long now, SSLEngine sslEngine) throws IOExcep * add a buffers to the writing queue * * @param buf buffer to be written - * @throws InterruptedException in case of interruption - * @throws IOException + * @throws IOException in case of io error */ public final void write(ByteBuffer buf) throws IOException { if (!key.isValid()) diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java index 96934b00..bb48b935 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOConnectionHandler.java @@ -8,6 +8,9 @@ import unknow.server.util.io.ByteBuffers; +/** + * handle nio connection event + */ @SuppressWarnings("unused") public interface NIOConnectionHandler { diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOLoop.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOLoop.java index ed1325c4..11205a94 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOLoop.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOLoop.java @@ -12,6 +12,7 @@ import org.slf4j.LoggerFactory; /** + * basic selector loop * @author unknow */ public class NIOLoop implements Runnable { diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOServerListener.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOServerListener.java index 9b75b126..154d425b 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOServerListener.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOServerListener.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; /** + * nio server listener * @author unknow */ public interface NIOServerListener { diff --git a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorkers.java b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorkers.java index efe41b1b..444ab3f7 100644 --- a/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorkers.java +++ b/unknow-server-nio/src/main/java/unknow/server/nio/NIOWorkers.java @@ -11,6 +11,7 @@ import unknow.server.nio.NIOServer.ConnectionFactory; /** + * represent a worker groups * @author unknow */ public interface NIOWorkers { diff --git a/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java b/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java index c7148fc8..397947cf 100644 --- a/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java +++ b/unknow-server-servlet/src/main/java/unknow/server/servlet/HttpConnection.java @@ -153,12 +153,7 @@ public final void execute(WorkerTask task) { } public void write(ByteBuffer buf) throws IOException { - try { - co.write(buf); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Failed to write", e); - } + co.write(buf); } public ServletContextImpl getCtx() { From c432de4ddcb62e1ac56f268ce34d7414ed3d6e36 Mon Sep 17 00:00:00 2001 From: Unknow Date: Mon, 13 Apr 2026 19:32:44 +0200 Subject: [PATCH 31/31] escape some more emoji --- .../java/unknow/server/bench/EncoderDecoder.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java index f6893990..51ce7c83 100644 --- a/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java +++ b/unknow-server-bench/src/main/java/unknow/server/bench/EncoderDecoder.java @@ -69,37 +69,37 @@ public class EncoderDecoder { private static final String COMPLEX = "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" - + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\uD83D\uDC68\u200D\uD83D\uDE80\uD83D\uDC69\u200D\uD83D\uDE80\uD83E\uDDD1\u200D\uD83D\uDE80 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" - + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\uD83D\uDC68\u200D\uD83D\uDE80\uD83D\uDC69\u200D\uD83D\uDE80\uD83E\uDDD1\u200D\uD83D\uDE80 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" - + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\uD83D\uDC68\u200D\uD83D\uDE80\uD83D\uDC69\u200D\uD83D\uDE80\uD83E\uDDD1\u200D\uD83D\uDE80 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" - + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\uD83D\uDC68\u200D\uD83D\uDE80\uD83D\uDC69\u200D\uD83D\uDE80\uD83E\uDDD1\u200D\uD83D\uDE80 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" - + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\uD83D\uDC68\u200D\uD83D\uDE80\uD83D\uDC69\u200D\uD83D\uDE80\uD83E\uDDD1\u200D\uD83D\uDE80 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱" + "𓂀𓆣𓇼𓁹𓃰 𐍈𐌰𐌽𐌳𐌴𐍂 🌌✨🌀💫 🌍🌎🌏 🧬⚛️🔮 ∑∏√∞ ≈≠≡ ⌘⌬⌭ ⟁\r\n" + "如来如去 如夢如幻 🌸🌺🌼 美しい世界 🌈🌠 🌕🌖🌗🌘\r\n" + "العربية العربية العربية ✨🌙⭐ ☯️☪️✡️ 🕉️ 🔱\r\n" + "\r\n" + "🐉🦄🐲🐾 🐉🦄🐲 \uD83E\uDDFF\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8\uFE0F 👁️👁️ 🌀🌀🌀\r\n" - + "👨‍🚀👩‍🚀🧑‍🚀 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + + "\uD83D\uDC68\u200D\uD83D\uDE80\uD83D\uDC69\u200D\uD83D\uDE80\uD83E\uDDD1\u200D\uD83D\uDE80 🚀🛸🚀🛸 🌌🌌🌌 💫💫💫\r\n" + "\r\n" + "𒀱𒂗𒆠 𒀭𒈹𒂊𒉡 𒌷𒆠𒂗𒄀 🏺🏺🏺\r\n" + "ᚠᛇᚻᚢᚦᚨᚱᚲ ⚔️🛡️ ⚔️🛡️\r\n" + "𑀓𑀺𑀢𑁆𑀢𑀺 𑀧𑀭𑀫𑀸𑀦 🌿🌿🌿\r\n" + "\r\n" + "🌲🍃🌲🍃🌲 🍄🍄🍄 🌸🌸🌸\r\n" + "🌊🌊🌊 🌊🌊🌊 🐚🐚🐚 🐠🐟🐡\r\n" + "\r\n" + "💻📡📶 🧠🤖🧠 🤖🧠🤖 ⚙️⚙️⚙️\r\n" + "0101 ✨ 𐍈 0101 ✨ 𐍈\r\n" + "\r\n" + "ქართული ქართული ქართული 🌄🌄\r\n" + "संस्कृतम् संस्कृतम् संस्कृतम् 🔱🔱\r\n" + "\r\n" + "🔥🔥🔥 ❄️❄️❄️ ⚡⚡⚡ 🌪️🌪️🌪️\r\n" + "❤️🧡💛💚💙💜🖤 🤍🤎 💖💗💓\r\n" + "\r\n" + "🌐🌐🌐 🌐🌐🌐 🌀🌀🌀 🌀🌀🌀\r\n" + "💫✨🌟⭐✨💫 🌟✨⭐💫✨🌟\r\n" + "\r\n" + "𓂀𓂀𓂀 𓇼𓇼𓇼 𓆣𓆣𓆣\r\n" + "𐍈𐍈𐍈 ᚠᚠᚠ 𒀱𒀱𒀱";