diff --git a/data/files/jsonserde.txt b/data/files/jsonserde.txt index 08963f2897b9..3fa8560cd578 100644 --- a/data/files/jsonserde.txt +++ b/data/files/jsonserde.txt @@ -1 +1 @@ -{"binarycolumn1" : -2, "binarycolumn2" : false, "binarycolumn3" : null, "binarycolumn4" : true, "binarycolumn5" : 1.23e45, "binarycolumn6" : "value", "booleancaseinsensitive" : "TrUE", "booleanstring" : "true", "booleanboolean" : true, "stringfalse" : "FaLSE", "somestring" : "somestringhere", "booleannull" : null, "booleannumfalse" : 0, "booleannumtrue" : -1} \ No newline at end of file +{"binarycolumn1" : -2, "binarycolumn2" : false, "binarycolumn3" : null, "binarycolumn4" : true, "binarycolumn5" : 1.23e45, "binarycolumn6" : "value", "booleancaseinsensitive" : "TrUE", "booleanstring" : "true", "booleanboolean" : true, "stringfalse" : "FaLSE", "somestring" : "somestringhere", "booleannull" : null, "booleannumfalse" : 0, "booleannumtrue" : -1, "decimalcol1" : -9999999999999999.99, "decimalcol2" : 9999999999999999.99, "decimalcol3" : 1000000000000000000000000000000000000.00, "decimalcol4" : 99.999 , "decimalcol5" : 1e39} \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/json_serde3.q b/ql/src/test/queries/clientpositive/json_serde3.q index 48e45bfa7699..4cb64e1d9598 100644 --- a/ql/src/test/queries/clientpositive/json_serde3.q +++ b/ql/src/test/queries/clientpositive/json_serde3.q @@ -2,6 +2,8 @@ drop table if exists json_serde3_1; drop table if exists json_serde3_2; drop table if exists json_serde3_3; drop table if exists json_serde3_4; +drop table if exists json_serde3_5; +drop table if exists json_serde3_6; create table json_serde3_1 ( binarycolumn1 binary, @@ -64,7 +66,35 @@ INSERT INTO TABLE json_serde3_4 VALUES ("TrUE", "true", true, "FaLSE", "somestri select * from json_serde3_4; +create table json_serde3_5 ( + decimalcol1 decimal(18,2), + decimalcol2 decimal(38,2), + decimalcol3 decimal(38,2), + decimalcol4 decimal(18,2), + decimalcol5 decimal(38,2)) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe'; + +LOAD DATA LOCAL INPATH '../../data/files/jsonserde.txt' into table json_serde3_5; +INSERT INTO TABLE json_serde3_5 VALUES (-9999999999999999.99, 9999999999999999.99, 1000000000000000000000000000000000000.00, 99.999, 1e39); + +select * from json_serde3_5; + +create table json_serde3_6 ( + decimalcol1 decimal(18,2), + decimalcol2 decimal(38,2), + decimalcol3 decimal(38,2), + decimalcol4 decimal(18,2), + decimalcol5 decimal(38,2)) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'; + +LOAD DATA LOCAL INPATH '../../data/files/jsonserde.txt' into table json_serde3_6; +INSERT INTO TABLE json_serde3_6 VALUES (-9999999999999999.99, 9999999999999999.99, 1000000000000000000000000000000000000.00, 99.999, 1e39); + +select * from json_serde3_6; + drop table json_serde3_1; drop table json_serde3_2; drop table json_serde3_3; -drop table json_serde3_4; \ No newline at end of file +drop table json_serde3_4; +drop table json_serde3_5; +drop table json_serde3_6; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/llap/json_serde3.q.out b/ql/src/test/results/clientpositive/llap/json_serde3.q.out index 0b33db51d621..b91ee954a6e6 100644 --- a/ql/src/test/results/clientpositive/llap/json_serde3.q.out +++ b/ql/src/test/results/clientpositive/llap/json_serde3.q.out @@ -22,6 +22,18 @@ PREHOOK: Output: database:default POSTHOOK: query: drop table if exists json_serde3_4 POSTHOOK: type: DROPTABLE POSTHOOK: Output: database:default +PREHOOK: query: drop table if exists json_serde3_5 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists json_serde3_5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default +PREHOOK: query: drop table if exists json_serde3_6 +PREHOOK: type: DROPTABLE +PREHOOK: Output: database:default +POSTHOOK: query: drop table if exists json_serde3_6 +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: database:default PREHOOK: query: create table json_serde3_1 ( binarycolumn1 binary, binarycolumn2 binary, @@ -77,7 +89,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@json_serde3_1 #### A masked pattern was here #### -2 FALSE NULL TRUE 1.23E45 value --2 false NULL true 1.23E45 value +-2 false NULL true 1.23E+45 value PREHOOK: query: create table json_serde3_2 ( binarycolumn1 binary, binarycolumn2 binary, @@ -131,7 +143,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@json_serde3_2 #### A masked pattern was here #### -2 FALSE NULL TRUE 1.23E45 value --2 false NULL true 1.23E45 value +-2 false NULL true 1.23E+45 value PREHOOK: query: create table json_serde3_3 ( booleancaseinsensitive boolean, booleanstring boolean, @@ -252,6 +264,108 @@ POSTHOOK: Input: default@json_serde3_4 #### A masked pattern was here #### true true true false true NULL false true true true true false true NULL false true +PREHOOK: query: create table json_serde3_5 ( + decimalcol1 decimal(18,2), + decimalcol2 decimal(38,2), + decimalcol3 decimal(38,2), + decimalcol4 decimal(18,2), + decimalcol5 decimal(38,2)) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde3_5 +POSTHOOK: query: create table json_serde3_5 ( + decimalcol1 decimal(18,2), + decimalcol2 decimal(38,2), + decimalcol3 decimal(38,2), + decimalcol4 decimal(18,2), + decimalcol5 decimal(38,2)) + row format serde 'org.apache.hadoop.hive.serde2.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde3_5 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/jsonserde.txt' into table json_serde3_5 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@json_serde3_5 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/jsonserde.txt' into table json_serde3_5 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@json_serde3_5 +PREHOOK: query: INSERT INTO TABLE json_serde3_5 VALUES (-9999999999999999.99, 9999999999999999.99, 1000000000000000000000000000000000000.00, 99.999, 1e39) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@json_serde3_5 +POSTHOOK: query: INSERT INTO TABLE json_serde3_5 VALUES (-9999999999999999.99, 9999999999999999.99, 1000000000000000000000000000000000000.00, 99.999, 1e39) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@json_serde3_5 +POSTHOOK: Lineage: json_serde3_5.decimalcol1 SCRIPT [] +POSTHOOK: Lineage: json_serde3_5.decimalcol2 SCRIPT [] +POSTHOOK: Lineage: json_serde3_5.decimalcol3 SCRIPT [] +POSTHOOK: Lineage: json_serde3_5.decimalcol4 SCRIPT [] +POSTHOOK: Lineage: json_serde3_5.decimalcol5 SCRIPT [] +PREHOOK: query: select * from json_serde3_5 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_serde3_5 +#### A masked pattern was here #### +POSTHOOK: query: select * from json_serde3_5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_serde3_5 +#### A masked pattern was here #### +-9999999999999999.99 9999999999999999.99 NULL 100.00 NULL +-9999999999999999.99 9999999999999999.99 NULL 100.00 NULL +PREHOOK: query: create table json_serde3_6 ( + decimalcol1 decimal(18,2), + decimalcol2 decimal(38,2), + decimalcol3 decimal(38,2), + decimalcol4 decimal(18,2), + decimalcol5 decimal(38,2)) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde3_6 +POSTHOOK: query: create table json_serde3_6 ( + decimalcol1 decimal(18,2), + decimalcol2 decimal(38,2), + decimalcol3 decimal(38,2), + decimalcol4 decimal(18,2), + decimalcol5 decimal(38,2)) + row format serde 'org.apache.hive.hcatalog.data.JsonSerDe' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde3_6 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/jsonserde.txt' into table json_serde3_6 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@json_serde3_6 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/jsonserde.txt' into table json_serde3_6 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@json_serde3_6 +PREHOOK: query: INSERT INTO TABLE json_serde3_6 VALUES (-9999999999999999.99, 9999999999999999.99, 1000000000000000000000000000000000000.00, 99.999, 1e39) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@json_serde3_6 +POSTHOOK: query: INSERT INTO TABLE json_serde3_6 VALUES (-9999999999999999.99, 9999999999999999.99, 1000000000000000000000000000000000000.00, 99.999, 1e39) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@json_serde3_6 +POSTHOOK: Lineage: json_serde3_6.decimalcol1 SCRIPT [] +POSTHOOK: Lineage: json_serde3_6.decimalcol2 SCRIPT [] +POSTHOOK: Lineage: json_serde3_6.decimalcol3 SCRIPT [] +POSTHOOK: Lineage: json_serde3_6.decimalcol4 SCRIPT [] +POSTHOOK: Lineage: json_serde3_6.decimalcol5 SCRIPT [] +PREHOOK: query: select * from json_serde3_6 +PREHOOK: type: QUERY +PREHOOK: Input: default@json_serde3_6 +#### A masked pattern was here #### +POSTHOOK: query: select * from json_serde3_6 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@json_serde3_6 +#### A masked pattern was here #### +-9999999999999999.99 9999999999999999.99 NULL 100.00 NULL +-9999999999999999.99 9999999999999999.99 NULL 100.00 NULL PREHOOK: query: drop table json_serde3_1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@json_serde3_1 @@ -292,3 +406,23 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@json_serde3_4 POSTHOOK: Output: database:default POSTHOOK: Output: default@json_serde3_4 +PREHOOK: query: drop table json_serde3_5 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@json_serde3_5 +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde3_5 +POSTHOOK: query: drop table json_serde3_5 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@json_serde3_5 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde3_5 +PREHOOK: query: drop table json_serde3_6 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@json_serde3_6 +PREHOOK: Output: database:default +PREHOOK: Output: default@json_serde3_6 +POSTHOOK: query: drop table json_serde3_6 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@json_serde3_6 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@json_serde3_6 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java index c504b9ec4ff2..da9721cc7666 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java @@ -37,6 +37,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fasterxml.jackson.databind.DeserializationFeature; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.common.type.Date; @@ -56,6 +57,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; import org.apache.hive.common.util.TimestampParser; @@ -172,7 +175,7 @@ public HiveJsonReader(ObjectInspector oi) { public HiveJsonReader(ObjectInspector oi, TimestampParser tsParser) { this.tsParser = tsParser; this.oi = oi; - this.objectMapper = new ObjectMapper(); + this.objectMapper = new ObjectMapper().enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); } /** @@ -426,7 +429,8 @@ private Object visitLeafNode(final JsonNode leafNode, case TIMESTAMP: return tsParser.parseTimestamp(leafNode.asText()); case DECIMAL: - return HiveDecimal.create(leafNode.asText()); + HiveDecimal decimal = HiveDecimal.create(leafNode.asText()); + return HiveDecimalUtils.enforcePrecisionScale(decimal, (DecimalTypeInfo) typeInfo); case TIMESTAMPLOCALTZ: final Timestamp ts = tsParser.parseTimestamp(leafNode.asText()); final ZoneId zid = ((TimestampLocalTZTypeInfo) typeInfo).timeZone();