diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 5a6d1a08663..db8f25015d8 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -136,6 +136,7 @@ import org.opensearch.sql.expression.function.PPLFuncImpTable; import org.opensearch.sql.expression.parse.RegexCommonUtils; import org.opensearch.sql.utils.ParseUtils; +import org.opensearch.sql.utils.WildcardRenameUtils; public class CalciteRelNodeVisitor extends AbstractNodeVisitor { @@ -485,27 +486,52 @@ public RelNode visitRename(Rename node, CalcitePlanContext context) { visitChildren(node, context); List originalNames = context.relBuilder.peek().getRowType().getFieldNames(); List newNames = new ArrayList<>(originalNames); + for (org.opensearch.sql.ast.expression.Map renameMap : node.getRenameList()) { - if (renameMap.getTarget() instanceof Field) { - Field t = (Field) renameMap.getTarget(); - String newName = t.getField().toString(); - RexNode check = rexVisitor.analyze(renameMap.getOrigin(), context); - if (check instanceof RexInputRef) { - RexInputRef ref = (RexInputRef) check; - newNames.set(ref.getIndex(), newName); - } else { - throw new SemanticCheckException( - String.format("the original field %s cannot be resolved", renameMap.getOrigin())); - } - } else { + if (!(renameMap.getTarget() instanceof Field)) { throw new SemanticCheckException( String.format("the target expected to be field, but is %s", renameMap.getTarget())); } + + String sourcePattern = ((Field) renameMap.getOrigin()).getField().toString(); + String targetPattern = ((Field) renameMap.getTarget()).getField().toString(); + + if (WildcardRenameUtils.isWildcardPattern(sourcePattern) + && !WildcardRenameUtils.validatePatternCompatibility(sourcePattern, targetPattern)) { + throw new SemanticCheckException( + "Source and target patterns have different wildcard counts"); + } + + List matchingFields = WildcardRenameUtils.matchFieldNames(sourcePattern, newNames); + + for (String fieldName : matchingFields) { + String newName = + WildcardRenameUtils.applyWildcardTransformation( + sourcePattern, targetPattern, fieldName); + if (newNames.contains(newName) && !newName.equals(fieldName)) { + removeFieldIfExists(newName, newNames, context); + } + int fieldIndex = newNames.indexOf(fieldName); + if (fieldIndex != -1) { + newNames.set(fieldIndex, newName); + } + } + + if (matchingFields.isEmpty() && newNames.contains(targetPattern)) { + removeFieldIfExists(targetPattern, newNames, context); + context.relBuilder.rename(newNames); + } } context.relBuilder.rename(newNames); return context.relBuilder.peek(); } + private void removeFieldIfExists( + String fieldName, List newNames, CalcitePlanContext context) { + newNames.remove(fieldName); + context.relBuilder.projectExcept(context.relBuilder.field(fieldName)); + } + @Override public RelNode visitSort(Sort node, CalcitePlanContext context) { visitChildren(node, context); diff --git a/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java new file mode 100644 index 00000000000..36912baed2c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/utils/WildcardRenameUtils.java @@ -0,0 +1,141 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** Utility class for handling wildcard patterns in rename operations. */ +public class WildcardRenameUtils { + + /** + * Check if pattern contains any supported wildcards. + * + * @param pattern the pattern to check + * @return true if pattern contains * wildcards + */ + public static boolean isWildcardPattern(String pattern) { + return pattern.contains("*"); + } + + /** + * Check if pattern is only wildcards that matches all fields. + * + * @param pattern the pattern to check + * @return true if pattern is only made up of wildcards "*" + */ + public static boolean isFullWildcardPattern(String pattern) { + return pattern.matches("\\*+"); + } + + /** + * Convert wildcard pattern to regex. + * + * @param pattern the wildcard pattern + * @return regex pattern with capture groups + */ + public static String wildcardToRegex(String pattern) { + String[] parts = pattern.split("\\*", -1); + return Arrays.stream(parts).map(Pattern::quote).collect(Collectors.joining("(.*)")); + } + + /** + * Match field names against wildcard pattern. + * + * @param wildcardPattern the pattern to match against + * @param availableFields collection of available field names + * @return list of matching field names + */ + public static List matchFieldNames( + String wildcardPattern, Collection availableFields) { + // Single wildcard matches all available fields + if (isFullWildcardPattern(wildcardPattern)) { + return new ArrayList<>(availableFields); + } + + String regexPattern = "^" + wildcardToRegex(wildcardPattern) + "$"; + Pattern pattern = Pattern.compile(regexPattern); + + return availableFields.stream() + .filter(field -> pattern.matcher(field).matches()) + .collect(Collectors.toList()); + } + + /** + * Apply wildcard transformation to get new field name. + * + * @param sourcePattern the source wildcard pattern + * @param targetPattern the target wildcard pattern + * @param actualFieldName the actual field name to transform + * @return transformed field name + * @throws IllegalArgumentException if patterns don't match or are invalid + */ + public static String applyWildcardTransformation( + String sourcePattern, String targetPattern, String actualFieldName) { + + if (sourcePattern.equals(targetPattern)) { + return actualFieldName; + } + + if (!isFullWildcardPattern(sourcePattern) || !isFullWildcardPattern(targetPattern)) { + if (sourcePattern.matches(".*\\*{2,}.*") || targetPattern.matches(".*\\*{2,}.*")) { + throw new IllegalArgumentException("Consecutive wildcards in pattern are not supported"); + } + } + + String sourceRegex = "^" + wildcardToRegex(sourcePattern) + "$"; + Matcher matcher = Pattern.compile(sourceRegex).matcher(actualFieldName); + + if (!matcher.matches()) { + throw new IllegalArgumentException( + String.format("Field '%s' does not match pattern '%s'", actualFieldName, sourcePattern)); + } + + String result = targetPattern; + + for (int i = 1; i <= matcher.groupCount(); i++) { + String capturedValue = matcher.group(i); + + int index = result.indexOf("*"); + if (index >= 0) { + result = result.substring(0, index) + capturedValue + result.substring(index + 1); + } else { + throw new IllegalArgumentException( + "Target pattern has fewer wildcards than source pattern"); + } + } + + return result; + } + + /** + * Validate that source and target patterns have matching wildcard counts. + * + * @param sourcePattern the source pattern + * @param targetPattern the target pattern + * @return true if patterns are compatible + */ + public static boolean validatePatternCompatibility(String sourcePattern, String targetPattern) { + int sourceWildcards = countWildcards(sourcePattern); + int targetWildcards = countWildcards(targetPattern); + return sourceWildcards == targetWildcards; + } + + /** + * Count the number of wildcards in a pattern. + * + * @param pattern the pattern to analyze + * @return number of wildcard characters + */ + private static int countWildcards(String pattern) { + return (int) pattern.chars().filter(ch -> ch == '*').count(); + } +} diff --git a/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java new file mode 100644 index 00000000000..a2582fa58f8 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/utils/WildcardRenameUtilsTest.java @@ -0,0 +1,197 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.ImmutableSet; +import java.util.List; +import org.junit.jupiter.api.Test; + +class WildcardRenameUtilsTest { + + @Test + void testIsWildcardPattern() { + assertTrue(WildcardRenameUtils.isWildcardPattern("*name")); + assertTrue(WildcardRenameUtils.isWildcardPattern("prefix*suffix")); + assertTrue(WildcardRenameUtils.isWildcardPattern("*")); + assertFalse(WildcardRenameUtils.isWildcardPattern("name")); + assertFalse(WildcardRenameUtils.isWildcardPattern("")); + } + + @Test + void testIsFullWildcardPattern() { + assertTrue(WildcardRenameUtils.isFullWildcardPattern("*")); + assertTrue(WildcardRenameUtils.isFullWildcardPattern("**")); + assertTrue(WildcardRenameUtils.isFullWildcardPattern("***")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("*name")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("prefix*")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("name")); + assertFalse(WildcardRenameUtils.isFullWildcardPattern("*_*")); + } + + @Test + void testWildcardToRegex() { + assertEquals("\\Q\\E(.*)\\Qname\\E", WildcardRenameUtils.wildcardToRegex("*name")); + assertEquals("\\Qname\\E(.*)\\Q\\E", WildcardRenameUtils.wildcardToRegex("name*")); + assertEquals( + "\\Q\\E(.*)\\Q_\\E(.*)\\Q_field\\E", WildcardRenameUtils.wildcardToRegex("*_*_field")); + } + + @Test + void testMatchFieldNames() { + ImmutableSet availableFields = + ImmutableSet.of("firstname", "lastname", "age", "address", "fullname"); + + List nameFields = WildcardRenameUtils.matchFieldNames("*name", availableFields); + assertEquals(List.of("firstname", "lastname", "fullname"), nameFields); + List firstFields = WildcardRenameUtils.matchFieldNames("first*", availableFields); + assertEquals(List.of("firstname"), firstFields); + List allFields = WildcardRenameUtils.matchFieldNames("*", availableFields); + assertEquals(List.of("firstname", "lastname", "age", "address", "fullname"), allFields); + } + + @Test + void testMatchFieldNamesNoMatches() { + ImmutableSet availableFields = + ImmutableSet.of("firstname", "lastname", "age", "address", "fullname"); + List noMatch = WildcardRenameUtils.matchFieldNames("*xyz", availableFields); + assertTrue(noMatch.isEmpty()); + } + + @Test + void testMatchFieldNamesNoWildcards() { + ImmutableSet availableFields = + ImmutableSet.of("firstname", "lastname", "age", "address", "fullname"); + List exactMatch = WildcardRenameUtils.matchFieldNames("age", availableFields); + assertEquals(List.of("age"), exactMatch); + List exactNoMatch = WildcardRenameUtils.matchFieldNames("xyz", availableFields); + assertTrue(exactNoMatch.isEmpty()); + } + + @Test + void testApplyWildcardTransformation() { + assertEquals( + "firstNAME", + WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "firstname")); + assertEquals( + "FIRSTname", + WildcardRenameUtils.applyWildcardTransformation("first*", "FIRST*", "firstname")); + assertEquals( + "user_profile", + WildcardRenameUtils.applyWildcardTransformation("*_*_field", "*_*", "user_profile_field")); + assertEquals( + "prefixfirst", + WildcardRenameUtils.applyWildcardTransformation("*name", "prefix*", "firstname")); + } + + @Test + void testFullWildcardTransformation() { + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); + assertEquals( + "new_firstname", + WildcardRenameUtils.applyWildcardTransformation("*", "new_*", "firstname")); + assertEquals( + "first", WildcardRenameUtils.applyWildcardTransformation("*name", "*", "firstname")); + } + + @Test + void testPartialMatchWildcardTransformation() { + assertEquals( + "FiRsTname", + WildcardRenameUtils.applyWildcardTransformation("f*r*tname", "F*R*Tname", "firstname")); + } + + @Test + void testApplyWildcardTransformationPatternMismatch() { + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("*name", "*NAME", "age")); + } + + @Test + void testValidatePatternCompatibility() { + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*name", "*NAME")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*_*", "*_*")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("prefix*suffix", "PREFIX*SUFFIX")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("name", "NAME")); + } + + @Test + void testValidatePatternCompatibilityFullWildcard() { + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "new_*")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("*", "*_old")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("old_*", "*")); + } + + @Test + void testValidatePatternCompatibilityInvalidPattern() { + assertFalse(WildcardRenameUtils.validatePatternCompatibility("*name", "*_*")); + assertFalse(WildcardRenameUtils.validatePatternCompatibility("*_*", "*")); + } + + @Test + void testFullWildcardPatternTransformation() { + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("*", "*", "firstname")); + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("**", "**", "firstname")); + assertEquals( + "firstname", WildcardRenameUtils.applyWildcardTransformation("***", "***", "firstname")); + } + + @Test + void testSingleWildcardFullPatternTransformation() { + assertEquals( + "firstname_suffix", + WildcardRenameUtils.applyWildcardTransformation("*", "*_suffix", "firstname")); + assertEquals( + "prefix_firstname", + WildcardRenameUtils.applyWildcardTransformation("*", "prefix_*", "firstname")); + assertEquals( + "prefix_firstname_suffix", + WildcardRenameUtils.applyWildcardTransformation("*", "prefix_*_suffix", "firstname")); + } + + @Test + void testMultipleWildcardSourcePatternError() { + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("**", "**_suffix", "firstname")); + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("***", "prefix_***", "firstname")); + } + + @Test + void testValidatePatternCompatibilityMultipleWildcards() { + assertTrue(WildcardRenameUtils.validatePatternCompatibility("**", "**")); + assertTrue(WildcardRenameUtils.validatePatternCompatibility("***", "***")); + assertFalse(WildcardRenameUtils.validatePatternCompatibility("**", "*")); + assertFalse(WildcardRenameUtils.validatePatternCompatibility("*", "**")); + } + + @Test + void testConsecutiveWildcardsError() { + assertThrows( + IllegalArgumentException.class, + () -> WildcardRenameUtils.applyWildcardTransformation("*n**me", "*N**ME", "firstname")); + assertThrows( + IllegalArgumentException.class, + () -> + WildcardRenameUtils.applyWildcardTransformation("**name", "**something", "firstname")); + assertThrows( + IllegalArgumentException.class, + () -> + WildcardRenameUtils.applyWildcardTransformation( + "**name**", "**something**", "firstname")); + } +} diff --git a/docs/category.json b/docs/category.json index 8ff175ee074..ee63e25e360 100644 --- a/docs/category.json +++ b/docs/category.json @@ -20,7 +20,6 @@ "user/ppl/cmd/parse.rst", "user/ppl/cmd/patterns.rst", "user/ppl/cmd/rare.rst", - "user/ppl/cmd/rename.rst", "user/ppl/cmd/search.rst", "user/ppl/cmd/sort.rst", "user/ppl/cmd/syntax.rst", @@ -62,6 +61,7 @@ "user/ppl/cmd/regex.rst", "user/ppl/cmd/rename.rst", "user/ppl/cmd/rex.rst", + "user/ppl/cmd/rename.rst", "user/ppl/cmd/stats.rst", "user/ppl/cmd/timechart.rst" ] diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index c942884248e..f5f129ca8f4 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -18,8 +18,23 @@ Syntax ============ rename AS ["," AS ]... -* source-field: mandatory. The name of the field you want to rename. -* field list: mandatory. The name you want to rename to. +* source-field: mandatory. The name of the field you want to rename. Supports wildcard patterns since version 3.3 using ``*``. +* target-field: mandatory. The name you want to rename to. Must have same number of wildcards as the source. + +Field Rename Behavior (Since version 3.3) +========================================== + +The rename command handles non-existent fields as follows: + +* **Renaming a non-existent field to a non-existent field**: No change occurs to the result set. +* **Renaming a non-existent field to an existing field**: The existing target field is removed from the result set. +* **Renaming an existing field to an existing field**: The existing target field is removed and the source field is renamed to the target. + + +**Notes:** + +* Literal asterisk (*) characters in field names cannot be replaced as asterisk is used for wildcard matching. +* Wildcards are only supported when the Calcite query engine is enabled. Example 1: Rename one field @@ -59,6 +74,64 @@ PPL query:: | 18 | null | +----+---------+ + +Example 3: Rename with wildcards +================================= + +The example shows renaming multiple fields using wildcard patterns. (Requires Calcite query engine) + +PPL query:: + + os> source=accounts | rename *name as *_name | fields first_name, last_name; + fetched rows / total rows = 4/4 + +------------+-----------+ + | first_name | last_name | + |------------+-----------| + | Amber | Duke | + | Hattie | Bond | + | Nanette | Bates | + | Dale | Adams | + +------------+-----------+ + + +Example 4: Rename with multiple wildcard patterns +================================================== + +The example shows renaming multiple fields using multiple wildcard patterns. (Requires Calcite query engine) + +PPL query:: + + os> source=accounts | rename *name as *_name, *_number as *number | fields first_name, last_name, accountnumber; + fetched rows / total rows = 4/4 + +------------+-----------+---------------+ + | first_name | last_name | accountnumber | + |------------+-----------+---------------| + | Amber | Duke | 1 | + | Hattie | Bond | 6 | + | Nanette | Bates | 13 | + | Dale | Adams | 18 | + +------------+-----------+---------------+ + +Example 5: Rename existing field to existing field +==================================== + +The example shows renaming an existing field to an existing field. The target field gets removed and the source field is renamed to the target field. + + +PPL query:: + + os> source=accounts | rename firstname as age | fields age; + fetched rows / total rows = 4/4 + +---------+ + | age | + |---------| + | Amber | + | Hattie | + | Nanette | + | Dale | + +---------+ + + Limitation ========== The ``rename`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java index a76bb298b26..93c74b2be52 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRenameIT.java @@ -63,19 +63,6 @@ public void testRefRenamedField() { "field [age] not found; input fields are: [country, month, year, name, state, renamed_age, _id, _index, _score, _maxscore, _sort, _routing]"); } - @Test - public void testRenameNotExistedField() throws IOException { - Throwable e = - assertThrowsWithReplace( - IllegalArgumentException.class, - () -> - executeQuery( - String.format( - "source = %s | rename renamed_age as age", TEST_INDEX_STATE_COUNTRY))); - verifyNotFoundAndInputFields(e.getMessage(), - "field [renamed_age] not found; input fields are: [country, month, year, name, state, age, _id, _index, _score, _maxscore, _sort, _routing]"); - } - @Test public void testRenameToMetaField() throws IOException { Throwable e = @@ -163,6 +150,237 @@ public void testRenameWithBackticksInAgg() throws IOException { verifyDataRows(result, rows(22.5, "Canada"), rows(50.0, "USA")); } + @Test + public void testRenameWildcardFields() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | fields name, country, state, month, year, age | rename *ame as *AME", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("nAME", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testRenameMultipleWildcardFields() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | fields name, country, state, month, year, age | rename *nt* as *NT*", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("couNTry", "string"), + schema("year", "int"), + schema("moNTh", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testRenameWildcardPrefix() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | fields name, country, state, month, year, age | rename *me as new_*", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("new_na", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testRenameFullWildcard() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, age | rename * as old_*", TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("old_name", "string"), schema("old_age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } + + @Test + public void testRenameMultipleWildcards() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | fields name, country, state, month, year, age | rename m*n*h as M*N*H", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("MoNtH", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testMultipleRenameWithWildcard() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, age | rename name as user_name | rename user_name as" + + " final_name", + TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("final_name", "string"), schema("age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } + + @Test + public void testChainedRename() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, age | rename name as user_name, user_name as" + + " final_name", + TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("final_name", "string"), schema("age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } + + @Test + public void testChainedRenameWithWildcard() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, age | rename *ame as *_ame, *_ame as *_AME", + TEST_INDEX_STATE_COUNTRY)); + verifySchema(result, schema("n_AME", "string"), schema("age", "int")); + verifyDataRows(result, rows("Jake", 70), rows("Hello", 30), rows("John", 25), rows("Jane", 20)); + } + + @Test + public void testRenamingToExistingField() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | fields name, country, state, month, year | rename name as age", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("age", "string"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023), + rows("Hello", "USA", "New York", 4, 2023), + rows("John", "Canada", "Ontario", 4, 2023), + rows("Jane", "Canada", "Quebec", 4, 2023)); + } + + @Test + public void testRenamingNonExistentField() throws IOException { + JSONObject result = + executeQuery( + String.format("source = %s | fields name, country, state, month, year, age | rename none as nothing", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testRenamingNonExistentFieldToExistingField() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | fields name, country, state, month, year | rename none as age", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023), + rows("Hello", "USA", "New York", 4, 2023), + rows("John", "Canada", "Ontario", 4, 2023), + rows("Jane", "Canada", "Quebec", 4, 2023)); + } + + @Test + public void testWildcardPatternDifferentCounts() { + Throwable e = + assertThrowsWithReplace( + IllegalArgumentException.class, + () -> + executeQuery( + String.format("source = %s | rename *a*e as *new", TEST_INDEX_STATE_COUNTRY))); + verifyErrorMessageContains(e, "Source and target patterns have different wildcard counts"); + } + + @Test + public void testRenameSameField() throws IOException { + JSONObject result = + executeQuery(String.format("source = %s | fields name, country, state, month, year, age | rename age as age", TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("name", "string"), + schema("age", "int"), + schema("state", "string"), + schema("country", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testMultipleRenameWithoutComma() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, country, state, month, year, age | rename name as user_name age as user_age country as location", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("user_name", "string"), + schema("user_age", "int"), + schema("state", "string"), + schema("location", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + @Test + public void testRenameMixedCommaAndSpace() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source = %s | fields name, country, state, month, year, age | rename name as user_name, age as user_age country as location", + TEST_INDEX_STATE_COUNTRY)); + verifySchema( + result, + schema("user_name", "string"), + schema("user_age", "int"), + schema("state", "string"), + schema("location", "string"), + schema("year", "int"), + schema("month", "int")); + verifyStandardDataRows(result); + } + + private void verifyStandardDataRows(JSONObject result) { + verifyDataRows( + result, + rows("Jake", "USA", "California", 4, 2023, 70), + rows("Hello", "USA", "New York", 4, 2023, 30), + rows("John", "Canada", "Ontario", 4, 2023, 25), + rows("Jane", "Canada", "Quebec", 4, 2023, 20)); + } + /** * Verify the error message for a not found field and the input fields. * This helper method makes tests insensitive to the order of input fields. diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java index 1844b5e07d7..d7f4a1ba8f3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java @@ -7,12 +7,10 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.util.MatcherUtils.columnName; -import static org.opensearch.sql.util.MatcherUtils.columnPattern; import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import java.io.IOException; import org.json.JSONObject; -import org.junit.Ignore; import org.junit.jupiter.api.Test; public class RenameCommandIT extends PPLIntegTestCase { @@ -42,13 +40,4 @@ public void testRenameMultiField() throws IOException { TEST_INDEX_ACCOUNT)); verifyColumn(result, columnName("FIRSTNAME"), columnName("AGE")); } - - @Ignore( - "Wildcard is unsupported yet. Enable once" - + " https://github.com/opensearch-project/sql/issues/787 is resolved.") - @Test - public void testRenameWildcardFields() throws IOException { - JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | rename %name as %NAME"); - verifyColumn(result, columnPattern(".*name$")); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java index 281455096d8..1cbd019eca3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java @@ -267,6 +267,32 @@ public void testCrossClusterRegexWithNegation() throws IOException { rows("Nanette")); } + @Test + public void testCrossClusterRenameWildcardPattern() throws IOException { + JSONObject result = + executeQuery( + String.format("search source=%s | rename *ame as *AME", TEST_INDEX_DOG_REMOTE)); + verifyColumn(result, columnName("dog_nAME"), columnName("holdersNAME"), columnName("age")); + verifySchema( + result, + schema("dog_nAME", "string"), + schema("holdersNAME", "string"), + schema("age", "bigint")); + } + + @Test + public void testCrossClusterRenameFullWildcard() throws IOException { + JSONObject result = + executeQuery(String.format("search source=%s | rename * as old_*", TEST_INDEX_DOG_REMOTE)); + verifyColumn( + result, columnName("old_dog_name"), columnName("old_holdersName"), columnName("old_age")); + verifySchema( + result, + schema("old_dog_name", "string"), + schema("old_holdersName", "string"), + schema("old_age", "bigint")); + } + @Test public void testCrossClusterRexBasic() throws IOException { JSONObject result = diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index b74143692fd..d53ad4eb149 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -155,7 +155,7 @@ wcFieldList ; renameCommand - : RENAME renameClasue (COMMA renameClasue)* + : RENAME renameClasue (COMMA? renameClasue)* ; statsCommand @@ -516,7 +516,7 @@ joinOption ; renameClasue - : orignalField = wcFieldExpression AS renamedField = wcFieldExpression + : orignalField = renameFieldExpression AS renamedField = renameFieldExpression ; byClause @@ -735,6 +735,11 @@ selectFieldExpression | STAR ; +renameFieldExpression + : wcQualifiedName + | STAR + ; + // functions evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index d6daa2744d6..8d924b77160 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -86,6 +86,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalOrContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalXorContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.MultiFieldRelevanceFunctionContext; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RenameFieldExpressionContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SingleFieldRelevanceFunctionContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SpanClauseContext; @@ -238,6 +239,14 @@ public UnresolvedExpression visitSelectFieldExpression( return new Field((QualifiedName) visit(ctx.wcQualifiedName())); } + @Override + public UnresolvedExpression visitRenameFieldExpression(RenameFieldExpressionContext ctx) { + if (ctx.STAR() != null) { + return new Field(QualifiedName.of("*")); + } + return new Field((QualifiedName) visit(ctx.wcQualifiedName())); + } + @Override public UnresolvedExpression visitSortField(SortFieldContext ctx) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 056bc4d23dd..6af3c72df31 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -119,6 +119,11 @@ public void testRenameCommandWithMultiFields() { anonymize("source=t | rename f as g,h as i,j as k")); } + @Test + public void testRenameCommandWithWildcards() { + assertEquals("source=t | rename f* as g*", anonymize("source=t | rename f* as g*")); + } + @Test public void testStatsCommandWithByClause() { assertEquals("source=t | stats count(a) by b", anonymize("source=t | stats count(a) by b"));