-
Notifications
You must be signed in to change notification settings - Fork 145
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[v3.0.0] Remove SparkSQL support (#3306)
* Remove SparkSQL support Signed-off-by: Tomoyuki Morita <[email protected]> * Remove a dependency Signed-off-by: Tomoyuki Morita <[email protected]> --------- Signed-off-by: Tomoyuki Morita <[email protected]>
- Loading branch information
Showing
53 changed files
with
226 additions
and
2,408 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
34 changes: 34 additions & 0 deletions
34
async-query-core/src/test/java/org/opensearch/sql/spark/data/type/SparkDataTypeTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.spark.data.type; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.*; | ||
|
|
||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class SparkDataTypeTest { | ||
|
|
||
| @Test | ||
| void testTypeName() { | ||
| String expectedTypeName = "spark_string"; | ||
| SparkDataType sparkDataType = new SparkDataType(expectedTypeName); | ||
|
|
||
| assertEquals( | ||
| expectedTypeName, sparkDataType.typeName(), "Type name should match the expected value"); | ||
| } | ||
|
|
||
| @Test | ||
| void testEqualsAndHashCode() { | ||
| SparkDataType type1 = new SparkDataType("spark_integer"); | ||
| SparkDataType type2 = new SparkDataType("spark_integer"); | ||
| SparkDataType type3 = new SparkDataType("spark_double"); | ||
|
|
||
| assertEquals(type1, type2); | ||
| assertNotEquals(type1, type3); | ||
| assertEquals(type1.hashCode(), type2.hashCode()); | ||
| assertNotEquals(type1.hashCode(), type3.hashCode()); | ||
| } | ||
| } |
40 changes: 40 additions & 0 deletions
40
async-query-core/src/test/java/org/opensearch/sql/spark/data/value/SparkExprValueTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.spark.data.value; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.*; | ||
|
|
||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.sql.spark.data.type.SparkDataType; | ||
|
|
||
| class SparkExprValueTest { | ||
| private final SparkDataType sparkDataType = new SparkDataType("char"); | ||
|
|
||
| @Test | ||
| public void getters() { | ||
| SparkExprValue sparkExprValue = new SparkExprValue(sparkDataType, "str"); | ||
|
|
||
| assertEquals(sparkDataType, sparkExprValue.type()); | ||
| assertEquals("str", sparkExprValue.value()); | ||
| } | ||
|
|
||
| @Test | ||
| public void unsupportedCompare() { | ||
| SparkExprValue sparkExprValue = new SparkExprValue(sparkDataType, "str"); | ||
|
|
||
| assertThrows(UnsupportedOperationException.class, () -> sparkExprValue.compare(sparkExprValue)); | ||
| } | ||
|
|
||
| @Test | ||
| public void testEquals() { | ||
| SparkExprValue sparkExprValue1 = new SparkExprValue(sparkDataType, "str"); | ||
| SparkExprValue sparkExprValue2 = new SparkExprValue(sparkDataType, "str"); | ||
| SparkExprValue sparkExprValue3 = new SparkExprValue(sparkDataType, "other"); | ||
|
|
||
| assertTrue(sparkExprValue1.equal(sparkExprValue2)); | ||
| assertFalse(sparkExprValue1.equal(sparkExprValue3)); | ||
| } | ||
| } |
63 changes: 63 additions & 0 deletions
63
...rg/opensearch/sql/spark/functions/response/DefaultSparkSqlFunctionResponseHandleTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.spark.functions.response; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.*; | ||
|
|
||
| import java.net.URL; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Paths; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import org.json.JSONObject; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.sql.data.model.ExprBooleanValue; | ||
| import org.opensearch.sql.data.model.ExprByteValue; | ||
| import org.opensearch.sql.data.model.ExprDateValue; | ||
| import org.opensearch.sql.data.model.ExprDoubleValue; | ||
| import org.opensearch.sql.data.model.ExprFloatValue; | ||
| import org.opensearch.sql.data.model.ExprIntegerValue; | ||
| import org.opensearch.sql.data.model.ExprLongValue; | ||
| import org.opensearch.sql.data.model.ExprShortValue; | ||
| import org.opensearch.sql.data.model.ExprStringValue; | ||
| import org.opensearch.sql.data.model.ExprTimestampValue; | ||
| import org.opensearch.sql.data.model.ExprValue; | ||
| import org.opensearch.sql.executor.ExecutionEngine; | ||
| import org.opensearch.sql.executor.ExecutionEngine.Schema.Column; | ||
|
|
||
| class DefaultSparkSqlFunctionResponseHandleTest { | ||
|
|
||
| @Test | ||
| public void testConstruct() throws Exception { | ||
| DefaultSparkSqlFunctionResponseHandle handle = | ||
| new DefaultSparkSqlFunctionResponseHandle(readJson()); | ||
|
|
||
| assertTrue(handle.hasNext()); | ||
| ExprValue value = handle.next(); | ||
| Map<String, ExprValue> row = value.tupleValue(); | ||
| assertEquals(ExprBooleanValue.of(true), row.get("col1")); | ||
| assertEquals(new ExprLongValue(2), row.get("col2")); | ||
| assertEquals(new ExprIntegerValue(3), row.get("col3")); | ||
| assertEquals(new ExprShortValue(4), row.get("col4")); | ||
| assertEquals(new ExprByteValue(5), row.get("col5")); | ||
| assertEquals(new ExprDoubleValue(6.1), row.get("col6")); | ||
| assertEquals(new ExprFloatValue(7.1), row.get("col7")); | ||
| assertEquals(new ExprTimestampValue("2024-01-02 03:04:05.1234"), row.get("col8")); | ||
| assertEquals(new ExprDateValue("2024-01-03 04:05:06.1234"), row.get("col9")); | ||
| assertEquals(new ExprStringValue("some string"), row.get("col10")); | ||
|
|
||
| ExecutionEngine.Schema schema = handle.schema(); | ||
| List<Column> columns = schema.getColumns(); | ||
| assertEquals("col1", columns.get(0).getName()); | ||
| } | ||
|
|
||
| private JSONObject readJson() throws Exception { | ||
| final URL url = | ||
| DefaultSparkSqlFunctionResponseHandle.class.getResource( | ||
| "/spark_execution_result_test.json"); | ||
| return new JSONObject(Files.readString(Paths.get(url.toURI()))); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
79 changes: 79 additions & 0 deletions
79
async-query-core/src/test/resources/spark_execution_result_test.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| { | ||
| "data" : { | ||
| "schema": [ | ||
| { | ||
| "column_name": "col1", | ||
| "data_type": "boolean" | ||
| }, | ||
| { | ||
| "column_name": "col2", | ||
| "data_type": "long" | ||
| }, | ||
| { | ||
| "column_name": "col3", | ||
| "data_type": "integer" | ||
| }, | ||
| { | ||
| "column_name": "col4", | ||
| "data_type": "short" | ||
| }, | ||
| { | ||
| "column_name": "col5", | ||
| "data_type": "byte" | ||
| }, | ||
| { | ||
| "column_name": "col6", | ||
| "data_type": "double" | ||
| }, | ||
| { | ||
| "column_name": "col7", | ||
| "data_type": "float" | ||
| }, | ||
| { | ||
| "column_name": "col8", | ||
| "data_type": "timestamp" | ||
| }, | ||
| { | ||
| "column_name": "col9", | ||
| "data_type": "date" | ||
| }, | ||
| { | ||
| "column_name": "col10", | ||
| "data_type": "string" | ||
| }, | ||
| { | ||
| "column_name": "col11", | ||
| "data_type": "other" | ||
| }, | ||
| { | ||
| "column_name": "col12", | ||
| "data_type": "other object" | ||
| }, | ||
| { | ||
| "column_name": "col13", | ||
| "data_type": "other array" | ||
| }, | ||
| { | ||
| "column_name": "col14", | ||
| "data_type": "other" | ||
| } | ||
| ], | ||
| "result": [ | ||
| { | ||
| "col1": true, | ||
| "col2": 2, | ||
| "col3": 3, | ||
| "col4": 4, | ||
| "col5": 5, | ||
| "col6": 6.1, | ||
| "col7": 7.1, | ||
| "col8": "2024-01-02 03:04:05.1234", | ||
| "col9": "2024-01-03 04:05:06.1234", | ||
| "col10": "some string", | ||
| "col11": "other value", | ||
| "col12": { "hello": "world" }, | ||
| "col13": [1, 2, 3] | ||
| } | ||
| ] | ||
| } | ||
| } |
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.