Fix NULL handling for Variant/Dynamic

client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatSerializer.java

@@ -207,7 +207,7 @@ public static boolean writeValuePreamble(OutputStream out, boolean defaultsSuppo
                 } else if (dataType == ClickHouseDataType.Array) {//If the column is an array
                     SerializerUtils.writeNonNull(out);//Then we send nonNull
                 } else if (dataType == ClickHouseDataType.Dynamic) {
-                    // do nothing
+                    SerializerUtils.writeNonNull(out);
                 } else {
                     throw new IllegalArgumentException(String.format("An attempt to write null into not nullable column '%s'", column));
                 }

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java

@@ -886,7 +886,10 @@ public ArrayValue readNested(ClickHouseColumn column) throws IOException {
     }
 
     public Object readVariant(ClickHouseColumn column) throws IOException {
-        int ordNum = readByte();
+        int ordNum = readByte() & 0xFF;
+        if (ordNum == 0xFF) {
+            return null;
+        }
         return readValue(column.getNestedColumns().get(ordNum));
     }
 

client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java

@@ -1,13 +1,13 @@
 package com.clickhouse.client.api.data_formats.internal;
 
+import com.clickhouse.data.ClickHouseColumn;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.lang.reflect.Array;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
-import java.util.Arrays;
 import java.util.TimeZone;
 
 import org.testng.Assert;
@@ -190,4 +190,18 @@ public void testArrayValue() throws Exception {
         Object[] array2 = array.getArrayOfObjects();
         Assert.assertEquals(array1.length, array2.length);
     }
+
+    @Test
+    public void testReadNullVariantReturnsNull() throws Exception {
+        ClickHouseColumn column = ClickHouseColumn.of("v", "Variant(Int32, String)");
+        BinaryStreamReader reader = new BinaryStreamReader(
+                new ByteArrayInputStream(new byte[]{(byte) 0xFF}),
+                TimeZone.getTimeZone("UTC"),
+                null,
+                new BinaryStreamReader.CachingByteBufferAllocator(),
+                false,
+                null);
+
+        Assert.assertNull(reader.readValue(column));
+    }
 }

client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java

@@ -291,6 +291,14 @@
         private Object field;
     }
 
+    @Data
+    @AllArgsConstructor
+    @NoArgsConstructor
+    public static class DTOForVariantNullPojoTests {
+        private int rowId;
+        private Object value;
+    }
+
     @Test(groups = {"integration"})
     public void testVariantWithDecimals() throws Exception {
         testVariantWith("decimals", new String[]{"field Variant(String, Decimal(4, 4))"},
@@ -313,6 +321,56 @@
                 });
     }
 
+    @Test(groups = {"integration"})
+    public void testVariantNullLiteral() throws Exception {
+        if (isVersionMatch("(,24.8]")) {
+            return;
+        }
+
+        List<GenericRecord> records = client.queryAll(
+                "SELECT NULL::Variant(Int32, String) AS val",
+                new QuerySettings().serverSetting("allow_experimental_variant_type", "1"));
+
+        Assert.assertEquals(records.size(), 1);
+        Assert.assertNull(records.get(0).getObject("val"));
+    }
+
+    @Test(groups = {"integration"})
+    public void testVariantNullLiteralWithPojo() throws Exception {
+        if (isVersionMatch("(,24.8]")) {
+            return;
+        }
+
+        final String table = "test_variant_null_literal_pojo";
+        CommandSettings createTableSettings = (CommandSettings) new CommandSettings()
+                .serverSetting("allow_experimental_variant_type", "1");
+        QuerySettings querySettings = new QuerySettings().serverSetting("allow_experimental_variant_type", "1");
+
+        client.execute("DROP TABLE IF EXISTS " + table).get();
+        client.execute(tableDefinition(table,
+                "rowId Int32",
+                "value Variant(Int32, String)"), createTableSettings).get();
+
+        try (QueryResponse ignored = client.query(
+                "INSERT INTO " + table
+                        + " SELECT 1 AS rowId, NULL::Variant(Int32, String) AS value"
+                        + " UNION ALL SELECT 2, 42::Variant(Int32, String)"
+                        + " UNION ALL SELECT 3, 'hello'::Variant(Int32, String)",
+                querySettings).get()) {
+        }
+
+        TableSchema tableSchema = client.getTableSchema(table);
+        client.register(DTOForVariantNullPojoTests.class, tableSchema);
+
+        List<DTOForVariantNullPojoTests> items =
+                client.queryAll("SELECT * FROM " + table + " ORDER BY rowId", DTOForVariantNullPojoTests.class, tableSchema);
+
+        Assert.assertEquals(items, Arrays.asList(
+                new DTOForVariantNullPojoTests(1, null),
+                new DTOForVariantNullPojoTests(2, 42),
+                new DTOForVariantNullPojoTests(3, "hello")));
+    }
+
     @Test(groups = {"integration"})
     public void testVariantWithArrays() throws Exception {
         testVariantWith("arrays", new String[]{"field Variant(String, Array(String))"},
@@ -811,13 +869,61 @@
         Assert.assertEquals(val, 3);
     }
 
+    @Test(groups = {"integration"})
+    public void testDynamicNullWithDefaultsColumn() throws Exception {
+        if (isVersionMatch("(,24.8]")) {
+            return;
+        }
+
+        final String table = "test_dynamic_null_defaults";
+        CommandSettings createTableSettings = (CommandSettings) new CommandSettings()
+                .serverSetting("allow_experimental_dynamic_type", "1");
+
+        client.execute("DROP TABLE IF EXISTS " + table).get();
+        client.execute(tableDefinition(table,
+                "rowId Int32",
+                "dyn Dynamic",
+                "extra String DEFAULT 'default_val'"), createTableSettings).get();
+
+        client.register(DTOForDynamicDefaultsTests.class, client.getTableSchema(table));
+
+        List<DTOForDynamicDefaultsTests> data = Arrays.asList(
+                new DTOForDynamicDefaultsTests(1, "hello", "explicit"),
+                new DTOForDynamicDefaultsTests(2, null, "after_null"),
+                new DTOForDynamicDefaultsTests(3, 42L, "last"));
+        client.insert(table, data).get().close();
+
+        List<GenericRecord> records = client.queryAll("SELECT rowId, dyn, extra FROM " + table + " ORDER BY rowId");
+        Assert.assertEquals(records.size(), data.size());
+
+        Assert.assertEquals(records.get(0).getInteger("rowId"), 1);
+        Assert.assertEquals(records.get(0).getString("dyn"), "hello");
+        Assert.assertEquals(records.get(0).getString("extra"), "explicit");
+
+        Assert.assertEquals(records.get(1).getInteger("rowId"), 2);
+        Assert.assertNull(records.get(1).getObject("dyn"));
+        Assert.assertEquals(records.get(1).getString("extra"), "after_null");
+
+        Assert.assertEquals(records.get(2).getInteger("rowId"), 3);
+        Assert.assertEquals(records.get(2).getString("dyn"), "42");
+        Assert.assertEquals(records.get(2).getString("extra"), "last");
+    }
+
     @Data
     @AllArgsConstructor
     public static class DTOForDynamicPrimitivesTests {
         private int rowId;
         private Object field;
     }
 
+    @Data
+    @AllArgsConstructor
+    public static class DTOForDynamicDefaultsTests {
+        private int rowId;
+        private Object dyn;
+        private String extra;
+    }
+
     @Test(groups = {"integration"})
     public void testAllDataTypesKnown() {
         List<GenericRecord> dbTypes = client.queryAll("SELECT * FROM system.data_type_families");

client-v2/src/test/java/com/clickhouse/client/internal/SerializerUtilsTests.java

@@ -1,8 +1,12 @@
 package com.clickhouse.client.internal;
 
+import com.clickhouse.client.api.data_formats.RowBinaryFormatSerializer;
 import com.clickhouse.client.api.data_formats.internal.SerializerUtils;
+import com.clickhouse.data.ClickHouseColumn;
 import org.testng.annotations.Test;
 
+import java.io.ByteArrayOutputStream;
+
 import org.testng.Assert;
 
 public class SerializerUtilsTests {
@@ -13,4 +17,15 @@ public void testConvertToInteger() {
         Assert.assertEquals(SerializerUtils.convertToInteger("1640995199").intValue(), expected);
         Assert.assertEquals(SerializerUtils.convertToInteger(false).intValue(), 0);
     }
+
+    @Test
+    public void testDynamicNullWithDefaultsWritesPreambleAndNothingTag() throws Exception {
+        ClickHouseColumn column = ClickHouseColumn.of("dyn", "Dynamic");
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+        Assert.assertTrue(RowBinaryFormatSerializer.writeValuePreamble(out, true, column, null));
+        SerializerUtils.serializeData(out, null, column);
+
+        Assert.assertEquals(out.toByteArray(), new byte[]{0, 0});
+    }
 }