Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/Azure/azure-kusto-spark i…
Browse files Browse the repository at this point in the history
…nto 2.4
  • Loading branch information
asaharn committed Dec 14, 2022
2 parents a132cd1 + 40a0790 commit 7128d15
Show file tree
Hide file tree
Showing 6 changed files with 600 additions and 16 deletions.
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ link your application with the artifact below to use the Azure Data Explorer Con
```
groupId = com.microsoft.azure.kusto
artifactId = kusto-spark_3.0_2.12
version = 3.1.5
version = 3.1.7
```

**In Maven**:

Look for the following coordinates:
```
com.microsoft.azure.kusto:kusto-spark_3.0_2.12:3.1.5
com.microsoft.azure.kusto:kusto-spark_3.0_2.12:3.1.7
```

Or clone this repository and build it locally to add it to your local maven repository,.
Expand All @@ -49,15 +49,15 @@ The jar can also be found under the [released package](https://github.com/Azure/
<dependency>
<groupId>com.microsoft.azure.kusto</groupId>
<artifactId>kusto-spark_3.0_2.12</artifactId>
<version>3.1.5</version>
<version>3.1.7</version>
</dependency>
```

**In SBT**:

```scala
libraryDependencies ++= Seq(
"com.microsoft.azure.kusto" %% "kusto-spark_3.0" % "3.1.5"
"com.microsoft.azure.kusto" %% "kusto-spark_3.0" % "3.1.7"
)
```

Expand All @@ -66,7 +66,7 @@ libraryDependencies ++= Seq(
Libraries -> Install New -> Maven -> copy the following coordinates:

```
com.microsoft.azure.kusto:kusto-spark_3.0_2.12:3.1.5
com.microsoft.azure.kusto:kusto-spark_3.0_2.12:3.1.7
```

#### Building Samples Module
Expand Down
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
package com.microsoft.kusto.spark.datasource

import java.sql.Timestamp
import java.util

import com.microsoft.azure.kusto.data.{KustoResultColumn, KustoResultSetTable, Results}
import com.microsoft.azure.kusto.data.KustoResultSetTable
import com.microsoft.kusto.spark.utils.DataTypeMapping
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
import org.apache.spark.sql.types.{StructType, _}
import org.joda.time.DateTime

import java.sql.Timestamp
import java.time.Instant
import java.util
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer

object KustoResponseDeserializer {
def apply(kustoResult: KustoResultSetTable): KustoResponseDeserializer = new KustoResponseDeserializer(kustoResult)
Expand All @@ -31,19 +29,24 @@ class KustoResponseDeserializer(val kustoResult: KustoResultSetTable) {
valueType.toLowerCase() match {
case "string" => value: Any => value
case "int64" => value: Any => value
case "datetime" => value: Any => new Timestamp(new DateTime(value).getMillis)
case "datetime" => value: Any => Timestamp.from(Instant.parse(value.toString))
case "timespan" => value: Any => value
case "sbyte" => value: Any => value
case "long" => value: Any => value match {
case "long" => {
case i: Int => i.toLong
case _ => value.asInstanceOf[Long]
case value => value.asInstanceOf[Long]
}
case "double" => value: Any => value
case "decimal" => value: Any => BigDecimal(value.asInstanceOf[String])
case "int" => value: Any => value
case "int32" => value: Any => value
case "bool" => value: Any => value
case "real" => value: Any => value
case "real" => {
case v: Int => v.toDouble
case v: Long => v.toDouble
case v: java.math.BigDecimal => v.doubleValue()
case v => v.asInstanceOf[Double]
}
case _ => value: Any => value.toString
}
}
Expand Down
272 changes: 272 additions & 0 deletions connector/src/test/resources/TestData/json/query-results-v1.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,272 @@
{
"Tables": [
{
"TableName": "Table_0",
"Columns": [
{
"ColumnName": "vnum",
"DataType": "Int32",
"ColumnType": "int"
},
{
"ColumnName": "vdec",
"DataType": "SqlDecimal",
"ColumnType": "decimal"
},
{
"ColumnName": "vdate",
"DataType": "DateTime",
"ColumnType": "datetime"
},
{
"ColumnName": "vspan",
"DataType": "TimeSpan",
"ColumnType": "timespan"
},
{
"ColumnName": "vobj",
"DataType": "Object",
"ColumnType": "dynamic"
},
{
"ColumnName": "vb",
"DataType": "SByte",
"ColumnType": "bool"
},
{
"ColumnName": "vreal",
"DataType": "Double",
"ColumnType": "real"
},
{
"ColumnName": "vstr",
"DataType": "String",
"ColumnType": "string"
},
{
"ColumnName": "vlong",
"DataType": "Int64",
"ColumnType": "long"
},
{
"ColumnName": "vguid",
"DataType": "Guid",
"ColumnType": "guid"
}
],
"Rows": [
[
0,
"2",
"2021-03-04T14:05:01.3109965Z",
null,
"'{\"key\": \"value\"}'",
false,
0.1111111111,
"'zxcv'",
9223372036854775806,
null
],
[
1,
"2.00000000000001",
"2020-03-04T14:05:01.3109965Z",
"01:23:45.6789000",
{
"key": "value"
},
true,
1111111111.1111111111,
"asdf",
9223372036854775807,
"74be27de-1e4e-49d9-b579-fe0b331d3642"
],
[
2,
"2",
"2020-03-01T00:05:01.4109965Z",
null,
"'{\"key\": \"value\"}'",
false,
1,
"'zxcv'",
9223372036854775806,
null
],
[
3,
"2",
"2021-04-04T14:05:01.3109965Z",
null,
"'{\"key\": \"value\"}'",
false,
9223372036854775806,
"'zxcv'",
9223372036854775806,
null
],
[
4,
"2",
"2022-03-04T14:05:01.3109965Z",
null,
"'{\"key\": \"value\"}'",
false,
9223372036854775806,
"'zxcv'",
9223372036854775806,
null
]
]
},
{
"TableName": "Table_1",
"Columns": [
{
"ColumnName": "Value",
"DataType": "String",
"ColumnType": "string"
}
],
"Rows": [
[
"{\"Visualization\":null,\"Title\":null,\"XColumn\":null,\"Series\":null,\"YColumns\":null,\"AnomalyColumns\":null,\"XTitle\":null,\"YTitle\":null,\"XAxis\":null,\"YAxis\":null,\"Legend\":null,\"YSplit\":null,\"Accumulate\":false,\"IsQuerySorted\":false,\"Kind\":null,\"Ymin\":\"NaN\",\"Ymax\":\"NaN\",\"Xmin\":null,\"Xmax\":null}"
]
]
},
{
"TableName": "Table_2",
"Columns": [
{
"ColumnName": "Timestamp",
"DataType": "DateTime",
"ColumnType": "datetime"
},
{
"ColumnName": "Severity",
"DataType": "Int32",
"ColumnType": "int"
},
{
"ColumnName": "SeverityName",
"DataType": "String",
"ColumnType": "string"
},
{
"ColumnName": "StatusCode",
"DataType": "Int32",
"ColumnType": "int"
},
{
"ColumnName": "StatusDescription",
"DataType": "String",
"ColumnType": "string"
},
{
"ColumnName": "Count",
"DataType": "Int32",
"ColumnType": "int"
},
{
"ColumnName": "RequestId",
"DataType": "Guid",
"ColumnType": "guid"
},
{
"ColumnName": "ActivityId",
"DataType": "Guid",
"ColumnType": "guid"
},
{
"ColumnName": "SubActivityId",
"DataType": "Guid",
"ColumnType": "guid"
},
{
"ColumnName": "ClientActivityId",
"DataType": "String",
"ColumnType": "string"
}
],
"Rows": [
[
"2022-12-14T07:04:52.9148512Z",
4,
"Info",
0,
"Query completed successfully",
1,
"2414f3e5-9c7d-4a0e-ba7d-3cbba616821a",
"2414f3e5-9c7d-4a0e-ba7d-3cbba616821a",
"63fbf587-7df7-4c4a-8a17-7642b13938e3",
"unspecified;5ef17816-08e1-49d4-ae17-4c3c73863e75"
],
[
"2022-12-14T07:04:52.9148512Z",
6,
"Stats",
0,
"{\"ExecutionTime\":0.0156089,\"resource_usage\":{\"cache\":{\"memory\":{\"hits\":0,\"misses\":0,\"total\":0},\"disk\":{\"hits\":0,\"misses\":0,\"total\":0},\"shards\":{\"hot\":{\"hitbytes\":2089,\"missbytes\":0,\"retrievebytes\":0},\"cold\":{\"hitbytes\":0,\"missbytes\":0,\"retrievebytes\":0},\"bypassbytes\":0}},\"cpu\":{\"user\":\"00:00:00\",\"kernel\":\"00:00:00\",\"total cpu\":\"00:00:00\"},\"memory\":{\"peak_per_node\":2624400},\"network\":{\"inter_cluster_total_bytes\":1115,\"cross_cluster_total_bytes\":0}},\"input_dataset_statistics\":{\"extents\":{\"total\":2,\"scanned\":2,\"scanned_min_datetime\":\"2022-11-15T12:53:51.6452197Z\",\"scanned_max_datetime\":\"2022-12-09T03:41:37.3226553Z\"},\"rows\":{\"total\":2,\"scanned\":100},\"rowstores\":{\"scanned_rows\":0,\"scanned_values_size\":0},\"shards\":{\"queries_generic\":2,\"queries_specialized\":0}},\"dataset_statistics\":[{\"table_row_count\":2,\"table_size\":213}],\"cross_cluster_resource_usage\":{}}",
1,
"2414f3e5-9c7d-4a0e-ba7d-3cbba616821a",
"2414f3e5-9c7d-4a0e-ba7d-3cbba616821a",
"63fbf587-7df7-4c4a-8a17-7642b13938e3",
"unspecified;5ef17816-08e1-49d4-ae17-4c3c73863e75"
]
]
},
{
"TableName": "Table_3",
"Columns": [
{
"ColumnName": "Ordinal",
"DataType": "Int64",
"ColumnType": "long"
},
{
"ColumnName": "Kind",
"DataType": "String",
"ColumnType": "string"
},
{
"ColumnName": "Name",
"DataType": "String",
"ColumnType": "string"
},
{
"ColumnName": "Id",
"DataType": "String",
"ColumnType": "string"
},
{
"ColumnName": "PrettyName",
"DataType": "String",
"ColumnType": "string"
}
],
"Rows": [
[
0,
"QueryResult",
"PrimaryResult",
"0eee0899-b744-49d9-96f6-464332a02e6b",
""
],
[
1,
"QueryProperties",
"@ExtendedProperties",
"382aae91-7828-4064-b262-491aad3e6c23",
""
],
[
2,
"QueryStatus",
"QueryStatus",
"00000000-0000-0000-0000-000000000000",
""
]
]
}
]
}
Loading

0 comments on commit 7128d15

Please sign in to comment.