Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions plugins/query-dsl-calcite/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ An OpenSearch plugin that converts OpenSearch Query DSL to Apache Calcite logica

This plugin integrates with OpenSearch's search pipeline to convert DSL queries into Calcite's logical plan representation. It supports:

- **Query Types**: Term, Range, Bool (must + filter), Match All
- **Query Types**: Term, Terms, Range, Bool (must + filter), Match All
- **Aggregations**: Metric (avg, sum, min, max, count) and bucket (terms, multi\_terms)
- **Sorting**: Pre-aggregation and post-aggregation sorting with BucketOrder
- **Pagination**: Offset and fetch (limit)
Expand Down Expand Up @@ -61,12 +61,14 @@ RelNode (Calcite Logical Plan)

### Queries

| DSL Query | Calcite Representation |
|-----------|------------------------|
| `term` | `=($field, value)` — equality filter |
| `range` (gte, lte, gt, lt) | `AND(>=($field, min), <=($field, max))` — range filter |
| `bool` (must + filter) | `AND(condition1, condition2, ...)` — flattened conjunction |
| `match_all` | Skipped (boolean literal `TRUE`) |
| DSL Query | Calcite Representation |
|-----------|---------------------------------------------------------------------------|
| `term` | `=($field, value)` — equality filter |
| `terms` | `SEARCH($field, Sarg[value1, value2, ...])` — multi-value equality filter |
| `range` (gte, lte, gt, lt) | `AND(>=($field, min), <=($field, max))` — range filter |
| `bool` (must + filter) | `AND(condition1, condition2, ...)` — flattened conjunction |
| `match_all` | Skipped (boolean literal `TRUE`) |
| `exists` | `IS NOT NULL($field)` — field existence check & boost not supported check |

### Aggregations

Expand Down Expand Up @@ -406,7 +408,7 @@ Current limitations:

1. **Read-only** — converts queries to logical plans but does not execute them
2. **Logging only** — converted plans are logged, not used for query execution
3. **Limited query types** — only `term`, `range`, `bool` (must + filter), and `match_all`
3. **Limited query types** — only `term`, `terms`, `range`, `bool` (must + filter), and `match_all`
4. **Bool query** — `should` and `must_not` clauses are not yet supported
5. **Nested objects** — flattened using dot notation, no true nested query support
6. **Pagination with aggregations** — `from`/`size` is not applied when aggregations are present
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,102 @@ public void testTermQueryConversion() throws Exception {
// Verify it's an equality condition
}

/**
* Test: Terms query conversion.
* Verifies that a terms query is converted to a LogicalFilter with SEARCH condition using Sarg.
*
* DSL Query:
* {
* "query": {
* "terms": {
* "category": ["electronics", "furniture"]
* }
* }
* }
*
* Expected Calcite Plan:
* LogicalFilter(condition=[SEARCH($0, Sarg['electronics':VARCHAR, 'furniture':VARCHAR]:VARCHAR)])
* LogicalTableScan(table=[[test-terms-query]])
*/
public void testTermsQueryConversion() throws Exception {
String indexName = "test-terms-query";
String mapping = "{"
+ "\"properties\": {"
+ " \"category\": {\"type\": \"keyword\"},"
+ " \"price\": {\"type\": \"long\"}"
+ "}"
+ "}";
client().admin().indices().prepareCreate(indexName)
.setMapping(mapping)
.get();
ensureGreen(indexName);

SearchSourceBuilder searchSource = new SearchSourceBuilder();
searchSource.query(QueryBuilders.termsQuery("category", "electronics", "furniture"));

SearchResponse response = convertDsl(searchSource, indexName);
assertNotNull("SearchResponse should not be null", response);

DslLogicalPlanPlugin plugin = getPlugin(DslLogicalPlanPlugin.class);
QueryPlans plans = plugin.getConverterService().convert(searchSource, indexName);
String plan = plans.get(QueryPlans.Type.HITS).get().relNode().toString();

assertTrue("Plan should contain LogicalFilter", plan.contains("LogicalFilter"));
assertTrue("Plan should contain SEARCH operator", plan.contains("SEARCH"));
assertTrue("Plan should contain Sarg", plan.contains("Sarg"));
assertTrue("Plan should contain 'electronics'", plan.contains("electronics"));
assertTrue("Plan should contain 'furniture'", plan.contains("furniture"));
}

/**
* Test: Terms query with non-default boost throws RuntimeException.
*/
public void testTermsQueryWithBoostThrowsException() throws Exception {
String indexName = "test-terms-boost";
String mapping = "{\"properties\": {\"category\": {\"type\": \"keyword\"}}}";
client().admin().indices().prepareCreate(indexName).setMapping(mapping).get();
ensureGreen(indexName);

SearchSourceBuilder searchSource = new SearchSourceBuilder();
searchSource.query(QueryBuilders.termsQuery("category", "electronics").boost(2.0f));

RuntimeException exception = expectThrows(RuntimeException.class, () -> convertDsl(searchSource, indexName));
assertThat(exception.getMessage(), containsString("does not support non-default boost"));
}

/**
* Test: Terms query with _name throws RuntimeException.
*/
public void testTermsQueryWithNameThrowsException() throws Exception {
String indexName = "test-terms-name";
String mapping = "{\"properties\": {\"category\": {\"type\": \"keyword\"}}}";
client().admin().indices().prepareCreate(indexName).setMapping(mapping).get();
ensureGreen(indexName);

SearchSourceBuilder searchSource = new SearchSourceBuilder();
searchSource.query(QueryBuilders.termsQuery("category", "electronics").queryName("my_query"));

RuntimeException exception = expectThrows(RuntimeException.class, () -> convertDsl(searchSource, indexName));
assertThat(exception.getMessage(), containsString("does not support _name"));
}

/**
* Test: Terms query with non-default value_type throws RuntimeException.
*/
public void testTermsQueryWithValueTypeThrowsException() throws Exception {
String indexName = "test-terms-valuetype";
String mapping = "{\"properties\": {\"category\": {\"type\": \"keyword\"}}}";
client().admin().indices().prepareCreate(indexName).setMapping(mapping).get();
ensureGreen(indexName);
Comment on lines +156 to +159
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this be done in the setup method of IT instead of every method?


SearchSourceBuilder searchSource = new SearchSourceBuilder();
searchSource.query(QueryBuilders.termsQuery("category", "electronics").valueType(
org.opensearch.index.query.TermsQueryBuilder.ValueType.BITMAP));

RuntimeException exception = expectThrows(RuntimeException.class, () -> convertDsl(searchSource, indexName));
assertThat(exception.getMessage(), containsString("does not support non-default value_type"));
}

/**
* Test: Range query conversion.
* Verifies that a range query is converted to a LogicalFilter with comparison operators.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ private QueryRegistryFactory() {}
public static QueryRegistry create() {
QueryRegistry registry = new QueryRegistry();
registry.register(new TermQueryTranslator());
registry.register(new TermsQueryTranslator());
registry.register(new RangeQueryTranslator());
registry.register(new MatchAllQueryTranslator());
registry.register(new BoolQueryTranslator(registry));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.dsl.query;

import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.opensearch.dsl.exception.ConversionException;
import org.opensearch.dsl.ConversionContext;
import org.opensearch.index.query.AbstractQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.TermsQueryBuilder;

import java.util.List;
import java.util.stream.Collectors;

/**
* Converts a {@link TermsQueryBuilder} to a Calcite IN RexNode.
*/
public class TermsQueryTranslator implements QueryTranslator {

@Override
public Class<? extends QueryBuilder> getQueryType() {
return TermsQueryBuilder.class;
}

@Override
public RexNode convert(QueryBuilder query, ConversionContext ctx) throws ConversionException {
ctx.requireOperatorSupported(SqlStdOperatorTable.IN);

TermsQueryBuilder termsQuery = (TermsQueryBuilder) query;

if (termsQuery.boost() != AbstractQueryBuilder.DEFAULT_BOOST) {
throw new RuntimeException("Terms query does not support non-default boost");
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to throw ConversionException to follow the same convention used in other places

}
if (termsQuery.queryName() != null) {
throw new RuntimeException("Terms query does not support _name");
}
if (termsQuery.valueType() != TermsQueryBuilder.ValueType.DEFAULT) {
throw new RuntimeException("Terms query does not support non-default value_type");
}

String fieldName = termsQuery.fieldName();
List<?> values = termsQuery.values();

if (values == null || values.isEmpty()) {
throw new RuntimeException("Terms query must have values");
}

RelDataTypeField field = ctx.getRowType().getField(fieldName, false, false);
if (field == null) {
throw new RuntimeException("Field '" + fieldName + "' not found in schema");
}

RexNode fieldRef = ctx.getRexBuilder().makeInputRef(field.getType(), field.getIndex());
List<RexNode> literals = values.stream()
.map(value -> ctx.getRexBuilder().makeLiteral(value, field.getType(), true))
.collect(Collectors.toList());

return ctx.getRexBuilder().makeIn(fieldRef, literals);
}
}