diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
index 0241202d99..94da2934f3 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractScalarTransformFunction.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.operator.transform.function;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.Configuration;
@@ -30,34 +31,50 @@
import java.util.List;
import java.util.Map;
import java.util.function.IntFunction;
+import javax.annotation.Nullable;
import org.apache.pinot.common.function.JsonPathCache;
import org.apache.pinot.core.operator.ColumnContext;
import org.apache.pinot.core.operator.blocks.ValueBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
+import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.core.util.NumberUtils;
import org.apache.pinot.core.util.NumericException;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.BooleanUtils;
import org.apache.pinot.spi.utils.JsonUtils;
+import org.apache.pinot.spi.utils.TimestampUtils;
+import org.roaringbitmap.RoaringBitmap;
-/**
- * The JsonExtractScalarTransformFunction class implements the json
- * path transformation based on
- * Stefan Goessner JsonPath
- * implementation..
- *
- * Please note, currently this method only works with String field. The values
- * in this field should be Json String.
- *
- * Usage:
- * jsonExtractScalar(jsonFieldName, 'jsonPath', 'resultsType')
- * jsonFieldName is the Json String field/expression.
- * jsonPath is a JsonPath expression which used to read from JSON
- * document
- * results_type refers to the results data type, could be INT,
- * LONG, FLOAT, DOUBLE, BIG_DECIMAL, STRING,
- * INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY, STRING_ARRAY.
- *
- */
+
+/// Implements the `jsonExtractScalar(jsonField, jsonPath, resultsType[, defaultValue])` transform.
+/// Reads a JSON document from `jsonField` for each row, resolves the
+/// [Stefan Goessner JsonPath](https://goessner.net/articles/JsonPath/) expression against it, and
+/// converts the resolved value to `resultsType`.
+///
+/// **Arguments:**
+/// - `jsonField` — single-value `STRING` or `BYTES` column / transform expression containing JSON.
+/// - `jsonPath` — JsonPath expression used to read the value.
+/// - `resultsType` — Pinot data type for the output. Append `_ARRAY` for multi-value results.
+/// - `defaultValue` (optional) — used when the path resolves to `null` or fails. Without it, unresolved
+/// SV rows throw `IllegalArgumentException`; MV rows surface as empty arrays, but null elements within
+/// a resolved array still throw.
+///
+/// **Supported `resultsType`:** `INT`, `LONG`, `FLOAT`, `DOUBLE`, `BIG_DECIMAL`, `BOOLEAN`, `TIMESTAMP`,
+/// `STRING`, `JSON`, `BYTES`, plus `_ARRAY` variants of `INT` / `LONG` / `FLOAT` / `DOUBLE` /
+/// `STRING`.
+///
+/// **Per-row coercion** of the JsonPath result to `resultsType`:
+/// - `BOOLEAN` (stored as `INT`) follows Pinot's numeric convention — any non-zero `Number` is true;
+/// `Boolean` and `"true"` / `"TRUE"` / `"1"` strings (via [BooleanUtils#toInt(String)]) are also true.
+/// - `TIMESTAMP` (stored as `LONG`) accepts numeric epoch millis directly; strings go through
+/// [TimestampUtils#toMillisSinceEpoch] (ISO-8601 and numeric millis strings).
+/// - `STRING` returns `String` values as-is; other JSON values are serialized via
+/// [JsonUtils#objectToString].
+/// - `BIG_DECIMAL` and `STRING` paths use a BigDecimal-preserving JSON parser
+/// (`JSON_PARSER_CONTEXT_WITH_BIG_DECIMAL`) to avoid precision loss on numeric values; other paths use
+/// the default parser.
+/// - Other types coerce via `Number` cast (preserved as the canonical primitive form) or
+/// `parse*(toString())`.
public class JsonExtractScalarTransformFunction extends BaseTransformFunction {
public static final String FUNCTION_NAME = "jsonExtractScalar";
@@ -77,7 +94,10 @@ public class JsonExtractScalarTransformFunction extends BaseTransformFunction {
private TransformFunction _jsonFieldTransformFunction;
private JsonPath _jsonPath;
+ private DataType _dataType;
+ private DataType _storedType;
private Object _defaultValue;
+ private boolean _defaultIsNull;
private TransformResultMetadata _resultMetadata;
@Override
@@ -86,8 +106,9 @@ public String getName() {
}
@Override
- public void init(List arguments, Map columnContextMap) {
- super.init(arguments, columnContextMap);
+ public void init(List arguments, Map columnContextMap,
+ QueryContext queryContext) {
+ super.init(arguments, columnContextMap, queryContext);
// Check that there are exactly 3 or 4 arguments
if (arguments.size() < 3 || arguments.size() > 4) {
throw new IllegalArgumentException(
@@ -106,20 +127,58 @@ public void init(List arguments, Map c
_jsonPath = JsonPathCache.INSTANCE.getOrCompute(jsonPathString);
String resultsType = ((LiteralTransformFunction) arguments.get(2)).getStringLiteral().toUpperCase();
boolean isSingleValue = !resultsType.endsWith("_ARRAY");
- DataType dataType;
try {
- dataType = DataType.valueOf(isSingleValue ? resultsType : resultsType.substring(0, resultsType.length() - 6));
+ _dataType = DataType.valueOf(isSingleValue ? resultsType : resultsType.substring(0, resultsType.length() - 6));
} catch (Exception e) {
throw new IllegalArgumentException(String.format(
"Unsupported results type: %s for jsonExtractScalar function. Supported types are: "
- + "INT/LONG/FLOAT/DOUBLE/BOOLEAN/BIG_DECIMAL/TIMESTAMP/STRING/INT_ARRAY/LONG_ARRAY/FLOAT_ARRAY"
- + "/DOUBLE_ARRAY/STRING_ARRAY",
- resultsType));
+ + "INT/LONG/FLOAT/DOUBLE/BIG_DECIMAL/BOOLEAN/TIMESTAMP/STRING/JSON/BYTES/"
+ + "INT_ARRAY/LONG_ARRAY/FLOAT_ARRAY/DOUBLE_ARRAY/STRING_ARRAY", resultsType));
}
+ _storedType = _dataType.getStoredType();
if (arguments.size() == 4) {
- _defaultValue = dataType.convert(((LiteralTransformFunction) arguments.get(3)).getStringLiteral());
+ LiteralTransformFunction literalTransformFun = (LiteralTransformFunction) arguments.get(3);
+ _defaultIsNull = literalTransformFun.isNull() && _nullHandlingEnabled;
+ switch (_dataType) {
+ case INT:
+ _defaultValue = literalTransformFun.getIntLiteral();
+ break;
+ case LONG:
+ _defaultValue = literalTransformFun.getLongLiteral();
+ break;
+ case FLOAT:
+ _defaultValue = literalTransformFun.getFloatLiteral();
+ break;
+ case DOUBLE:
+ _defaultValue = literalTransformFun.getDoubleLiteral();
+ break;
+ case BIG_DECIMAL:
+ _defaultValue = literalTransformFun.getBigDecimalLiteral();
+ break;
+ case BOOLEAN:
+ // Stored as Integer 0 / 1 to match BOOLEAN's storedType (INT) so per-row consumers can
+ // unbox directly without a Boolean → Integer conversion.
+ _defaultValue = literalTransformFun.getBooleanLiteral() ? 1 : 0;
+ break;
+ case TIMESTAMP:
+ // Use long literal so numeric millis stay exact and string timestamps use LiteralContext parsing.
+ _defaultValue = literalTransformFun.getLongLiteral();
+ break;
+ case STRING:
+ case JSON:
+ _defaultValue = literalTransformFun.getStringLiteral();
+ break;
+ case BYTES:
+ _defaultValue = literalTransformFun.getBytesLiteral();
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported results type: " + _dataType + " for jsonExtractScalar function. Supported types are: "
+ + "INT/LONG/FLOAT/DOUBLE/BIG_DECIMAL/BOOLEAN/TIMESTAMP/STRING/JSON/BYTES"
+ );
+ }
}
- _resultMetadata = new TransformResultMetadata(dataType, isSingleValue, false);
+ _resultMetadata = new TransformResultMetadata(_dataType, isSingleValue, false);
}
@Override
@@ -127,22 +186,50 @@ public TransformResultMetadata getResultMetadata() {
return _resultMetadata;
}
+ @Override
+ @Nullable
+ public RoaringBitmap getNullBitmap(ValueBlock valueBlock) {
+ if (!_defaultIsNull) {
+ return super.getNullBitmap(valueBlock);
+ }
+ RoaringBitmap bitmap = new RoaringBitmap();
+ for (TransformFunction arg : _arguments.subList(1, _arguments.size() - 1)) {
+ RoaringBitmap argBitmap = arg.getNullBitmap(valueBlock);
+ if (argBitmap != null) {
+ bitmap.or(argBitmap);
+ }
+ }
+ int numDocs = valueBlock.getNumDocs();
+ RoaringBitmap nullBitmap = new RoaringBitmap();
+ IntFunction