feat(parse_cbor): add parse_cbor function (#1152)

* Add cbor support

* Move json_type_def to separate module

---------

Co-authored-by: Semyon Uchvatov <[email protected]>

Cargo.toml

@@ -68,6 +68,7 @@ stdlib = [
   "dep:chacha20poly1305",
   "dep:charset",
   "dep:convert_case",
+  "dep:ciborium",
   "dep:cidr-utils",
   "dep:community-id",
   "dep:crc",
@@ -131,6 +132,7 @@ charset = { version = "0.1", optional = true }
 encoding_rs = { version = "0.8.35", optional = false }
 chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"], optional = true }
 chrono-tz = { version = "0.10", default-features = false, optional = true }
+ciborium = { version = "0.2.2", default-features = false, optional = true }
 cidr-utils = { version = "0.6", optional = true }
 csv = { version = "1", optional = true }
 clap = { version = "4", features = ["derive"], optional = true }

LICENSE-3rdparty.csv

@@ -44,6 +44,7 @@ chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly130
 charset,https://github.com/hsivonen/charset,Apache-2.0 OR MIT,Henri Sivonen <[email protected]>
 chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors
 chrono-tz,https://github.com/chronotope/chrono-tz,MIT OR Apache-2.0,The chrono-tz Authors
+ciborium,https://github.com/enarx/ciborium,Apache-2.0,Nathaniel McCallum <[email protected]>
 cidr,https://github.com/stbuehler/rust-cidr,MIT,Stefan Bühler <[email protected]>
 cidr-utils,https://github.com/magiclen/cidr-utils,MIT,Magic Len <[email protected]>
 cipher,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers
@@ -68,6 +69,7 @@ crc32fast,https://github.com/srijs/rust-crc32fast,MIT OR Apache-2.0,"Sam Rijs <s
 crossbeam-channel,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-channel Authors
 crossbeam-epoch,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-epoch Authors
 crossbeam-utils,https://github.com/crossbeam-rs/crossbeam,MIT OR Apache-2.0,The crossbeam-utils Authors
+crunchy,https://github.com/eira-fransham/crunchy,MIT,Vurich <[email protected]>
 crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers
 crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers
 csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant <[email protected]>
@@ -103,6 +105,7 @@ generic-array,https://github.com/fizyk20/generic-array,MIT,"Bartłomiej Kamińsk
 getrandom,https://github.com/rust-random/getrandom,MIT OR Apache-2.0,The Rand Project Developers
 gimli,https://github.com/gimli-rs/gimli,MIT OR Apache-2.0,The gimli Authors
 grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger <[email protected]>
+half,https://github.com/starkat99/half-rs,MIT OR Apache-2.0,Kathryn Long <[email protected]>
 hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Antras <[email protected]>
 heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors
 heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats <[email protected]>

changelog.d/1152.feature.md

@@ -0,0 +1 @@
+Add `parse_cbor` function

license-tool.toml

@@ -0,0 +1,2 @@
+[overrides]
+"crunchy" = { origin = "https://github.com/eira-fransham/crunchy" }

src/stdlib/json_utils/json_type_def.rs

@@ -0,0 +1,24 @@
+use crate::prelude::{Collection, TypeDef};
+use crate::value::Kind;
+
+pub(crate) fn json_inner_kind() -> Kind {
+    Kind::null()
+        | Kind::bytes()
+        | Kind::integer()
+        | Kind::float()
+        | Kind::boolean()
+        | Kind::array(Collection::any())
+        | Kind::object(Collection::any())
+}
+
+pub(crate) fn json_type_def() -> TypeDef {
+    TypeDef::bytes()
+        .fallible()
+        .or_boolean()
+        .or_integer()
+        .or_float()
+        .add_null()
+        .or_null()
+        .or_array(Collection::from_unknown(json_inner_kind()))
+        .or_object(Collection::from_unknown(json_inner_kind()))
+}

src/stdlib/json_utils/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod json_type_def;

src/stdlib/mod.rs

@@ -31,6 +31,7 @@ pub use wasm_unsupported_function::WasmUnsupportedFunction;
 
 use crate::compiler::Function;
 
+mod json_utils;
 mod string_utils;
 mod util;
 mod wasm_unsupported_function;
@@ -141,6 +142,7 @@ cfg_if::cfg_if! {
         mod parse_aws_cloudwatch_log_subscription_message;
         mod parse_aws_vpc_flow_log;
         mod parse_cef;
+        mod parse_cbor;
         mod parse_common_log;
         mod parse_csv;
         mod parse_duration;
@@ -324,6 +326,7 @@ cfg_if::cfg_if! {
         pub use parse_aws_alb_log::ParseAwsAlbLog;
         pub use parse_aws_cloudwatch_log_subscription_message::ParseAwsCloudWatchLogSubscriptionMessage;
         pub use parse_aws_vpc_flow_log::ParseAwsVpcFlowLog;
+        pub use parse_cbor::ParseCbor;
         pub use parse_cef::ParseCef;
         pub use parse_common_log::ParseCommonLog;
         pub use parse_csv::ParseCsv;
@@ -514,6 +517,7 @@ pub fn all() -> Vec<Box<dyn Function>> {
         Box::new(ParseAwsAlbLog),
         Box::new(ParseAwsCloudWatchLogSubscriptionMessage),
         Box::new(ParseAwsVpcFlowLog),
+        Box::new(ParseCbor),
         Box::new(ParseCef),
         Box::new(ParseCommonLog),
         Box::new(ParseCsv),

src/stdlib/parse_cbor.rs

@@ -0,0 +1,132 @@
+use crate::compiler::prelude::*;
+use crate::stdlib::json_utils::json_type_def::json_type_def;
+use ciborium::de::from_reader;
+use zstd::zstd_safe::WriteBuf;
+
+fn parse_cbor(value: Value) -> Resolved {
+    let bytes = value.try_bytes()?;
+    let value = from_reader(bytes.as_slice()).map_err(|e| format!("unable to parse cbor: {e}"))?;
+    Ok(value)
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct ParseCbor;
+
+impl Function for ParseCbor {
+    fn identifier(&self) -> &'static str {
+        "parse_cbor"
+    }
+
+    fn summary(&self) -> &'static str {
+        "parse a string to a CBOR type"
+    }
+
+    fn usage(&self) -> &'static str {
+        indoc! {"
+            Parses the provided `value` as CBOR.
+        "}
+    }
+
+    fn examples(&self) -> &'static [Example] {
+        &[
+            Example {
+                title: "object",
+                source: r#"parse_cbor!(decode_base64!("oWVmaWVsZGV2YWx1ZQ=="))"#,
+                result: Ok(r#"{ "field": "value" }"#),
+            },
+            Example {
+                title: "array",
+                source: r#"parse_cbor!(decode_base64!("gvUA"))"#,
+                result: Ok("[true, 0]"),
+            },
+            Example {
+                title: "string",
+                source: r#"parse_cbor!(decode_base64!("ZWhlbGxv"))"#,
+                result: Ok("hello"),
+            },
+            Example {
+                title: "integer",
+                source: r#"parse_cbor!(decode_base64!("GCo="))"#,
+                result: Ok("42"),
+            },
+            Example {
+                title: "float",
+                source: r#"parse_cbor!(decode_base64!("+0BFEKPXCj1x"))"#,
+                result: Ok("42.13"),
+            },
+            Example {
+                title: "boolean",
+                source: r#"parse_cbor!(decode_base64!("9A=="))"#,
+                result: Ok("false"),
+            },
+        ]
+    }
+
+    fn compile(
+        &self,
+        _state: &state::TypeState,
+        _ctx: &mut FunctionCompileContext,
+        arguments: ArgumentList,
+    ) -> Compiled {
+        let value = arguments.required("value");
+        Ok(ParseCborFn { value }.as_expr())
+    }
+
+    fn parameters(&self) -> &'static [Parameter] {
+        &[Parameter {
+            keyword: "value",
+            kind: kind::BYTES,
+            required: true,
+        }]
+    }
+}
+
+#[derive(Debug, Clone)]
+struct ParseCborFn {
+    value: Box<dyn Expression>,
+}
+
+impl FunctionExpression for ParseCborFn {
+    fn resolve(&self, ctx: &mut Context) -> Resolved {
+        let value = self.value.resolve(ctx)?;
+        parse_cbor(value)
+    }
+
+    fn type_def(&self, _: &state::TypeState) -> TypeDef {
+        json_type_def()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::value;
+    use nom::AsBytes;
+    use std::env;
+    use std::fs;
+    use std::path::PathBuf;
+
+    fn test_data_dir() -> PathBuf {
+        PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("tests/data/cbor")
+    }
+
+    fn read_cbor_file(cbor_bin_message_path: &str) -> Vec<u8> {
+        fs::read(test_data_dir().join(cbor_bin_message_path)).unwrap()
+    }
+
+    test_function![
+        parse_cbor => ParseCbor;
+
+        parses {
+            args: func_args![ value: value!(read_cbor_file("simple.cbor").as_bytes()) ],
+            want: Ok(value!({ field: "value" })),
+            tdef: json_type_def(),
+        }
+
+        complex_cbor {
+            args: func_args![ value: value!(read_cbor_file("complex.cbor").as_bytes()) ],
+            want: Ok(value!({ object: {string: "value", number: 42, array: ["hello", "world"], boolean: false} })),
+            tdef: json_type_def(),
+        }
+    ];
+}