diff --git a/data/files/sampleJson.json b/data/files/sampleJson.json new file mode 100644 index 000000000000..58b04421d450 --- /dev/null +++ b/data/files/sampleJson.json @@ -0,0 +1 @@ +{"tiny_value": 128, "small_value" : 32768, "int_value" : 2147483648, "big_value" : 9223372036854775808} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index 7a547763e363..ca2a60dba9ab 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -128,6 +128,11 @@ public ByteWritable evaluate(IntWritable i) { if (i == null) { return null; } else { + int value = i.get(); + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw new IllegalArgumentException("Value out of range for Byte: " + value); + } + byteWritable.set((byte) i.get()); return byteWritable; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index e6b985e1053a..49b2bfc59ae5 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -146,6 +146,10 @@ public IntWritable evaluate(LongWritable i) { if (i == null) { return null; } else { + long value = i.get(); + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Value out of range for Integer: " + value); + } intWritable.set((int) i.get()); return intWritable; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index c465b633f632..d8989c0e1f70 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -128,6 +128,10 @@ public ShortWritable evaluate(IntWritable i) { if (i == null) { return null; } else { + int value = i.get(); + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw new IllegalArgumentException("Value out of range for Short: " + value); + } shortWritable.set((short) i.get()); return shortWritable; } diff --git a/ql/src/test/queries/clientnegative/json_serde_neg1.q b/ql/src/test/queries/clientnegative/json_serde_neg1.q new file mode 100644 index 000000000000..d29baa8def4c --- /dev/null +++ b/ql/src/test/queries/clientnegative/json_serde_neg1.q @@ -0,0 +1,10 @@ +drop table if exists json_serde1_1; + +create table json_serde1_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'; + +insert into table json_serde1_1 values (128, 32768, 2147483648, 9223372036854775808); + +select * from json_serde1_1; + +drop table json_serde1_1; \ No newline at end of file diff --git a/ql/src/test/queries/clientnegative/json_serde_neg2.q b/ql/src/test/queries/clientnegative/json_serde_neg2.q new file mode 100644 index 000000000000..93e4301d2001 --- /dev/null +++ b/ql/src/test/queries/clientnegative/json_serde_neg2.q @@ -0,0 +1,10 @@ +drop table if exists json_serde2_1; + +create table json_serde2_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'; + +LOAD DATA LOCAL INPATH '../../data/files/sampleJson.json' INTO TABLE json_serde2_1; + +select * from json_serde2_1; + +drop table json_serde2_1; \ No newline at end of file diff --git a/ql/src/test/queries/clientnegative/json_serde_neg3.q b/ql/src/test/queries/clientnegative/json_serde_neg3.q new file mode 100644 index 000000000000..333182562e66 --- /dev/null +++ b/ql/src/test/queries/clientnegative/json_serde_neg3.q @@ -0,0 +1,8 @@ +drop table if exists json_serde3_1; + +create table json_serde3_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; + +insert into table json_serde3_1 values (127, 32768, 2147483648, 9223372036854775808); + +select * from json_serde3_1; \ No newline at end of file diff --git a/ql/src/test/queries/clientnegative/json_serde_neg4.q b/ql/src/test/queries/clientnegative/json_serde_neg4.q new file mode 100644 index 000000000000..132a7b0ed2af --- /dev/null +++ b/ql/src/test/queries/clientnegative/json_serde_neg4.q @@ -0,0 +1,8 @@ +drop table if exists json_serde4_1; + +create table json_serde4_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; + +LOAD DATA LOCAL INPATH '../../data/files/sampleJson.json' INTO TABLE json_serde4_1; + +select * from json_serde4_1; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/prepare_plan.q b/ql/src/test/queries/clientpositive/prepare_plan.q index c39ec866cfd9..2b3968ce5efc 100644 --- a/ql/src/test/queries/clientpositive/prepare_plan.q +++ b/ql/src/test/queries/clientpositive/prepare_plan.q @@ -198,8 +198,8 @@ prepare pbetween from select count(*) from alltypes where (c BETWEEN ? AND ?) AND (v BETWEEN ? AND ?) AND (d BETWEEN ? AND ?) AND (dt BETWEEN ? AND ?) OR (ctinyint BETWEEN ? AND ?) AND (csmallint BETWEEN ? AND ?) AND (cint BETWEEN ? AND ?) AND (cfloat BETWEEN ? AND ?) AND (cdouble BETWEEN ? AND ?) OR (cstring1 BETWEEN ? AND ?) AND (ctimestamp1 BETWEEN ? AND ?) OR (cbigint BETWEEN ? AND ?); -explain execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 1000, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133; -execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 1000, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133; +explain execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 127, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133; +execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 127, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133; DROP TABLE testParam; DROP TABLE alltypes; diff --git a/ql/src/test/queries/clientpositive/udf_to_byte.q b/ql/src/test/queries/clientpositive/udf_to_byte.q index 77811bb1431d..3b20a08695be 100644 --- a/ql/src/test/queries/clientpositive/udf_to_byte.q +++ b/ql/src/test/queries/clientpositive/udf_to_byte.q @@ -7,7 +7,7 @@ SELECT CAST(NULL AS TINYINT) FROM src tablesample (1 rows); SELECT CAST(TRUE AS TINYINT) FROM src tablesample (1 rows); SELECT CAST(CAST(-18 AS SMALLINT) AS TINYINT) FROM src tablesample (1 rows); -SELECT CAST(-129 AS TINYINT) FROM src tablesample (1 rows); +SELECT CAST(-127 AS TINYINT) FROM src tablesample (1 rows); SELECT CAST(CAST(-1025 AS BIGINT) AS TINYINT) FROM src tablesample (1 rows); SELECT CAST(CAST(-3.14 AS DOUBLE) AS TINYINT) FROM src tablesample (1 rows); diff --git a/ql/src/test/results/clientnegative/json_serde_neg1.q.out b/ql/src/test/results/clientnegative/json_serde_neg1.q.out new file mode 100644 index 000000000000..18d290cebe43 --- /dev/null +++ b/ql/src/test/results/clientnegative/json_serde_neg1.q.out @@ -0,0 +1,73 @@ +PREHOOK: query: drop table if exists json_serde1_1 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists json_serde1_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create table json_serde1_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde1_1 +POSTHOOK: query: create table json_serde1_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde1_1 +PREHOOK: query: insert into table json_serde1_1 values (128, 32768, 2147483648, 9223372036854775808) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@json_serde1_1 +Status: Failed +Vertex failed, vertexName=Map 1, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ByteWritable org.apache.hadoop.hive.ql.udf.UDFToByte.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Byte: 128 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Byte: 128 +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ByteWritable org.apache.hadoop.hive.ql.udf.UDFToByte.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Byte: 128 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Byte: 128 +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Map 1] killed/failed due to:OWN_TASK_FAILURE] +[Masked Vertex killed due to OTHER_VERTEX_FAILURE] +DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1 +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Map 1, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ByteWritable org.apache.hadoop.hive.ql.udf.UDFToByte.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Byte: 128 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Byte: 128 +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ByteWritable org.apache.hadoop.hive.ql.udf.UDFToByte.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Byte: 128 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Byte: 128 +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Map 1] killed/failed due to:OWN_TASK_FAILURE][Masked Vertex killed due to OTHER_VERTEX_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1 diff --git a/ql/src/test/results/clientnegative/json_serde_neg2.q.out b/ql/src/test/results/clientnegative/json_serde_neg2.q.out new file mode 100644 index 000000000000..abb9f813bcdd --- /dev/null +++ b/ql/src/test/results/clientnegative/json_serde_neg2.q.out @@ -0,0 +1,33 @@ +PREHOOK: query: drop table if exists json_serde2_1 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists json_serde2_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create table json_serde2_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde2_1 +POSTHOOK: query: create table json_serde2_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde2_1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/sampleJson.json' INTO TABLE json_serde2_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@json_serde2_1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/sampleJson.json' INTO TABLE json_serde2_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@json_serde2_1 +PREHOOK: query: select * from json_serde2_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_serde2_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from json_serde2_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_serde2_1 +#### A masked pattern was here #### +Failed with exception java.io.IOException:org.apache.hadoop.hive.serde2.SerDeException: java.lang.IllegalArgumentException: Failed to parse integral value for category BYTE: 128 diff --git a/ql/src/test/results/clientnegative/json_serde_neg3.q.out b/ql/src/test/results/clientnegative/json_serde_neg3.q.out new file mode 100644 index 000000000000..f3ca76f69bc7 --- /dev/null +++ b/ql/src/test/results/clientnegative/json_serde_neg3.q.out @@ -0,0 +1,73 @@ +PREHOOK: query: drop table if exists json_serde3_1 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists json_serde3_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create table json_serde3_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde3_1 +POSTHOOK: query: create table json_serde3_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde3_1 +PREHOOK: query: insert into table json_serde3_1 values (127, 32768, 2147483648, 9223372036854775808) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@json_serde3_1 +Status: Failed +Vertex failed, vertexName=Map 1, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ShortWritable org.apache.hadoop.hive.ql.udf.UDFToShort.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Short: 32768 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Short: 32768 +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ShortWritable org.apache.hadoop.hive.ql.udf.UDFToShort.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Short: 32768 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Short: 32768 +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Map 1] killed/failed due to:OWN_TASK_FAILURE] +[Masked Vertex killed due to OTHER_VERTEX_FAILURE] +DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1 +FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Map 1, vertexId=vertex_#ID#, diagnostics=[Task failed, taskId=task_#ID#, diagnostics=[TaskAttempt 0 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ShortWritable org.apache.hadoop.hive.ql.udf.UDFToShort.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Short: 32768 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Short: 32768 +#### A masked pattern was here #### +], TaskAttempt 1 failed, info=[Error: Error while running task ( failure ) : attempt_#ID#:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable +#### A masked pattern was here #### +Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to execute method public org.apache.hadoop.hive.serde2.io.ShortWritable org.apache.hadoop.hive.ql.udf.UDFToShort.evaluate(org.apache.hadoop.io.IntWritable):Value out of range for Short: 32768 +#### A masked pattern was here #### +Caused by: java.lang.reflect.InvocationTargetException +#### A masked pattern was here #### +Caused by: java.lang.IllegalArgumentException: Value out of range for Short: 32768 +#### A masked pattern was here #### +]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_#ID# [Map 1] killed/failed due to:OWN_TASK_FAILURE][Masked Vertex killed due to OTHER_VERTEX_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:1 diff --git a/ql/src/test/results/clientnegative/json_serde_neg4.q.out b/ql/src/test/results/clientnegative/json_serde_neg4.q.out new file mode 100644 index 000000000000..a9374130eff2 --- /dev/null +++ b/ql/src/test/results/clientnegative/json_serde_neg4.q.out @@ -0,0 +1,33 @@ +PREHOOK: query: drop table if exists json_serde4_1 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists json_serde4_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: create table json_serde4_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde4_1 +POSTHOOK: query: create table json_serde4_1 (tiny_value TINYINT, small_value SMALLINT, int_value INT, big_value BIGINT) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde4_1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/sampleJson.json' INTO TABLE json_serde4_1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@json_serde4_1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/sampleJson.json' INTO TABLE json_serde4_1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@json_serde4_1 +PREHOOK: query: select * from json_serde4_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_serde4_1 +#### A masked pattern was here #### +POSTHOOK: query: select * from json_serde4_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_serde4_1 +#### A masked pattern was here #### +Failed with exception java.io.IOException:org.apache.hadoop.hive.serde2.SerDeException: org.apache.hadoop.hive.serde2.SerDeException: java.lang.IllegalArgumentException: Failed to parse integral value for category BYTE: 128 diff --git a/ql/src/test/results/clientpositive/llap/prepare_plan.q.out b/ql/src/test/results/clientpositive/llap/prepare_plan.q.out index 6d1a0e180c77..264df5d4bd85 100644 --- a/ql/src/test/results/clientpositive/llap/prepare_plan.q.out +++ b/ql/src/test/results/clientpositive/llap/prepare_plan.q.out @@ -2381,11 +2381,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: explain execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 1000, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 +PREHOOK: query: explain execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 127, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 PREHOOK: type: EXECUTE QUERY PREHOOK: Input: default@alltypes #### A masked pattern was here #### -POSTHOOK: query: explain execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 1000, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 +POSTHOOK: query: explain execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 127, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 POSTHOOK: type: EXECUTE QUERY POSTHOOK: Input: default@alltypes #### A masked pattern was here #### @@ -2405,10 +2405,10 @@ STAGE PLANS: Map Operator Tree: TableScan alias: alltypes - filterExpr: ((c BETWEEN CAST( 'ch1' AS CHAR(5)) AND CAST( 'ch2' AS CHAR(5)) and v BETWEEN CAST( 'var1' AS varchar(10)) AND CAST( 'var2' AS varchar(10)) and d BETWEEN CAST( 1000.34 AS decimal(10,3)) AND CAST( 2000 AS decimal(10,3)) and dt BETWEEN CAST( '1947-12-12' AS DATE) AND CAST( '1968-12-31' AS DATE)) or (ctinyint BETWEEN UDFToByte(11) AND UDFToByte(1000) and csmallint BETWEEN UDFToShort(15601) AND UDFToShort(1) and cint BETWEEN UDFToInteger(788564623) AND UDFToInteger(23) and cfloat BETWEEN UDFToFloat(1) AND UDFToFloat(18) and cdouble BETWEEN UDFToDouble(0) AND UDFToDouble(15601)) or (cstring1 BETWEEN CAST( 'xTlDv24JYv4s' AS STRING) AND CAST( 'str1' AS STRING) and ctimestamp1 BETWEEN CAST( '1969-12-31 16:00:02.351' AS TIMESTAMP) AND CAST( '2020-12-31 16:00:01' AS TIMESTAMP)) or cbigint BETWEEN UDFToLong(0) AND UDFToLong(133)) (type: boolean) + filterExpr: ((c BETWEEN CAST( 'ch1' AS CHAR(5)) AND CAST( 'ch2' AS CHAR(5)) and v BETWEEN CAST( 'var1' AS varchar(10)) AND CAST( 'var2' AS varchar(10)) and d BETWEEN CAST( 1000.34 AS decimal(10,3)) AND CAST( 2000 AS decimal(10,3)) and dt BETWEEN CAST( '1947-12-12' AS DATE) AND CAST( '1968-12-31' AS DATE)) or (ctinyint BETWEEN UDFToByte(11) AND UDFToByte(127) and csmallint BETWEEN UDFToShort(15601) AND UDFToShort(1) and cint BETWEEN UDFToInteger(788564623) AND UDFToInteger(23) and cfloat BETWEEN UDFToFloat(1) AND UDFToFloat(18) and cdouble BETWEEN UDFToDouble(0) AND UDFToDouble(15601)) or (cstring1 BETWEEN CAST( 'xTlDv24JYv4s' AS STRING) AND CAST( 'str1' AS STRING) and ctimestamp1 BETWEEN CAST( '1969-12-31 16:00:02.351' AS TIMESTAMP) AND CAST( '2020-12-31 16:00:01' AS TIMESTAMP)) or cbigint BETWEEN UDFToLong(0) AND UDFToLong(133)) (type: boolean) Statistics: Num rows: 18346 Data size: 8590681 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator - predicate: ((c BETWEEN CAST( 'ch1' AS CHAR(5)) AND CAST( 'ch2' AS CHAR(5)) and v BETWEEN CAST( 'var1' AS varchar(10)) AND CAST( 'var2' AS varchar(10)) and d BETWEEN CAST( 1000.34 AS decimal(10,3)) AND CAST( 2000 AS decimal(10,3)) and dt BETWEEN CAST( '1947-12-12' AS DATE) AND CAST( '1968-12-31' AS DATE)) or (ctinyint BETWEEN UDFToByte(11) AND UDFToByte(1000) and csmallint BETWEEN UDFToShort(15601) AND UDFToShort(1) and cint BETWEEN UDFToInteger(788564623) AND UDFToInteger(23) and cfloat BETWEEN UDFToFloat(1) AND UDFToFloat(18) and cdouble BETWEEN UDFToDouble(0) AND UDFToDouble(15601)) or (cstring1 BETWEEN CAST( 'xTlDv24JYv4s' AS STRING) AND CAST( 'str1' AS STRING) and ctimestamp1 BETWEEN CAST( '1969-12-31 16:00:02.351' AS TIMESTAMP) AND CAST( '2020-12-31 16:00:01' AS TIMESTAMP)) or cbigint BETWEEN UDFToLong(0) AND UDFToLong(133)) (type: boolean) + predicate: ((c BETWEEN CAST( 'ch1' AS CHAR(5)) AND CAST( 'ch2' AS CHAR(5)) and v BETWEEN CAST( 'var1' AS varchar(10)) AND CAST( 'var2' AS varchar(10)) and d BETWEEN CAST( 1000.34 AS decimal(10,3)) AND CAST( 2000 AS decimal(10,3)) and dt BETWEEN CAST( '1947-12-12' AS DATE) AND CAST( '1968-12-31' AS DATE)) or (ctinyint BETWEEN UDFToByte(11) AND UDFToByte(127) and csmallint BETWEEN UDFToShort(15601) AND UDFToShort(1) and cint BETWEEN UDFToInteger(788564623) AND UDFToInteger(23) and cfloat BETWEEN UDFToFloat(1) AND UDFToFloat(18) and cdouble BETWEEN UDFToDouble(0) AND UDFToDouble(15601)) or (cstring1 BETWEEN CAST( 'xTlDv24JYv4s' AS STRING) AND CAST( 'str1' AS STRING) and ctimestamp1 BETWEEN CAST( '1969-12-31 16:00:02.351' AS TIMESTAMP) AND CAST( '2020-12-31 16:00:01' AS TIMESTAMP)) or cbigint BETWEEN UDFToLong(0) AND UDFToLong(133)) (type: boolean) Statistics: Num rows: 2266 Data size: 1061232 Basic stats: COMPLETE Column stats: COMPLETE Select Operator Statistics: Num rows: 2266 Data size: 1061232 Basic stats: COMPLETE Column stats: COMPLETE @@ -2447,11 +2447,11 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 1000, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 +PREHOOK: query: execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 127, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 PREHOOK: type: EXECUTE QUERY PREHOOK: Input: default@alltypes #### A masked pattern was here #### -POSTHOOK: query: execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 1000, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 +POSTHOOK: query: execute pbetween using 'ch1' ,'ch2' ,'var1' ,'var2',1000.34, 2000.0, '1947-12-12', '1968-12-31', 11, 127, 15601, 1, 788564623, 23,1.0, 18.00, 0, 15601.0, 'xTlDv24JYv4s', 'str1', '1969-12-31 16:00:02.351','2020-12-31 16:00:01', 0, 133 POSTHOOK: type: EXECUTE QUERY POSTHOOK: Input: default@alltypes #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/udf_to_byte.q.out b/ql/src/test/results/clientpositive/llap/udf_to_byte.q.out index 078c4760187d..30973f943300 100644 --- a/ql/src/test/results/clientpositive/llap/udf_to_byte.q.out +++ b/ql/src/test/results/clientpositive/llap/udf_to_byte.q.out @@ -25,15 +25,15 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -18 -PREHOOK: query: SELECT CAST(-129 AS TINYINT) FROM src tablesample (1 rows) +PREHOOK: query: SELECT CAST(-127 AS TINYINT) FROM src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src #### A masked pattern was here #### -POSTHOOK: query: SELECT CAST(-129 AS TINYINT) FROM src tablesample (1 rows) +POSTHOOK: query: SELECT CAST(-127 AS TINYINT) FROM src tablesample (1 rows) POSTHOOK: type: QUERY POSTHOOK: Input: default@src #### A masked pattern was here #### -127 +-127 PREHOOK: query: SELECT CAST(CAST(-1025 AS BIGINT) AS TINYINT) FROM src tablesample (1 rows) PREHOOK: type: QUERY PREHOOK: Input: default@src diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java index c504b9ec4ff2..ce7fb2088300 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java @@ -382,21 +382,19 @@ private Object visitLeafNode(final JsonNode leafNode, final ObjectInspector oi) throws SerDeException { final PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; final PrimitiveTypeInfo typeInfo = poi.getTypeInfo(); + final PrimitiveCategory category = typeInfo.getPrimitiveCategory(); if (typeInfo.getPrimitiveCategory() != PrimitiveCategory.STRING) { Preconditions.checkArgument(leafNode.getNodeType() != JsonNodeType.OBJECT); Preconditions.checkArgument(leafNode.getNodeType() != JsonNodeType.ARRAY); } - switch (typeInfo.getPrimitiveCategory()) { + switch (category) { case INT: - return Integer.valueOf(leafNode.asInt()); case BYTE: - return Byte.valueOf((byte) leafNode.asInt()); case SHORT: - return Short.valueOf((short) leafNode.asInt()); case LONG: - return Long.valueOf(leafNode.asLong()); + return parseIntegralValue(leafNode, category); case BOOLEAN: if ("false".equalsIgnoreCase(leafNode.asText())) { return Boolean.FALSE; @@ -445,6 +443,25 @@ private Object visitLeafNode(final JsonNode leafNode, } } + private Object parseIntegralValue(JsonNode leafNode, PrimitiveCategory category) { + try { + switch (category) { + case INT: + return Integer.valueOf(leafNode.asText()); + case BYTE: + return Byte.valueOf(leafNode.asText()); + case SHORT: + return Short.valueOf(leafNode.asText()); + case LONG: + return Long.valueOf(leafNode.asText()); + default: + throw new IllegalArgumentException("Unexpected category: " + category); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to parse integral value for category " + category + ": " + leafNode.asText(), e); + } + } + /** * A user may configure the encoding for binary data represented as text * within a JSON object. This method applies that encoding to the text.