diff --git a/Cargo.lock b/Cargo.lock index f401edf8..9ab6af51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,7 +190,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -201,7 +201,7 @@ checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -248,9 +248,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" dependencies = [ "serde", ] @@ -373,9 +373,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata", @@ -430,7 +430,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -447,9 +447,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.5" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" dependencies = [ "jobserver", "libc", @@ -512,9 +512,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", "clap_derive", @@ -531,9 +531,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstream", "anstyle", @@ -544,14 +544,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -718,7 +718,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "crossterm_winapi", "mio", "parking_lot", @@ -765,7 +765,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -776,7 +776,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -816,7 +816,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -893,7 +893,7 @@ checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -1035,7 +1035,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -1102,7 +1102,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -1130,7 +1130,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "ignore", "walkdir", ] @@ -1370,9 +1370,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -1414,9 +1414,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "lock_api" @@ -1498,7 +1498,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "cfg-if", "libc", ] @@ -1509,7 +1509,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "cfg-if", "cfg_aliases", "libc", @@ -1582,9 +1582,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "os_info" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb6651f4be5e39563c4fe5cc8326349eb99a25d805a3493f791d5bfd0269e430" +checksum = "6e6520c8cc998c5741ee68ec1dc369fc47e5f0ea5320018ecf2a1ccd6328f48b" dependencies = [ "log", "serde", @@ -1665,9 +1665,9 @@ checksum = "e3aeb8f54c078314c2065ee649a7241f46b9d8e418e1a9581ba0546657d7aa3a" [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -1780,9 +1780,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -1793,7 +1793,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "chrono", "flate2", "hex", @@ -1807,7 +1807,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "chrono", "hex", ] @@ -1841,9 +1841,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -1904,7 +1904,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", ] [[package]] @@ -1979,11 +1979,11 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "errno", "libc", "linux-raw-sys", @@ -1992,9 +1992,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -2034,14 +2034,14 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -2174,14 +2174,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] name = "symbolic-common" -version = "12.12.4" +version = "12.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd33e73f154e36ec223c18013f7064a2c120f1162fc086ac9933542def186b00" +checksum = "bf08b42a6f9469bd8584daee39a1352c8133ccabc5151ccccb15896ef047d99a" dependencies = [ "debugid", "memmap2", @@ -2191,9 +2191,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.12.4" +version = "12.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e51191290147f071777e37fe111800bb82a9059f9c95b19d2dd41bfeddf477" +checksum = "32f73b5a5bd4da72720c45756a2d11edf110116b87f998bda59b97be8c2c7cf1" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -2213,9 +2213,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.91" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -2224,12 +2224,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -2286,7 +2287,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2297,7 +2298,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2375,7 +2376,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2430,7 +2431,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2515,9 +2516,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "b913a3b5fe84142e269d63cc62b64319ccaf89b748fc31fe025177f767a756c4" dependencies = [ "getrandom", ] @@ -2603,34 +2604,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2638,28 +2640,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -2781,7 +2786,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2792,7 +2797,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] [[package]] @@ -2978,5 +2983,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.91", + "syn 2.0.96", ] diff --git a/brush-core/benches/shell.rs b/brush-core/benches/shell.rs index 0a94f6da..ed29401b 100644 --- a/brush-core/benches/shell.rs +++ b/brush-core/benches/shell.rs @@ -146,7 +146,9 @@ mod unix { #[cfg(unix)] criterion::criterion_group! { name = benches; - config = criterion::Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new(100, pprof::criterion::Output::Flamegraph(None))); + config = criterion::Criterion::default() + .measurement_time(std::time::Duration::from_secs(10)) + .with_profiler(pprof::criterion::PProfProfiler::new(100, pprof::criterion::Output::Flamegraph(None))); targets = unix::criterion_benchmark } #[cfg(unix)] diff --git a/brush-parser/benches/parser.rs b/brush-parser/benches/parser.rs index f7f90eb4..b33c3a35 100644 --- a/brush-parser/benches/parser.rs +++ b/brush-parser/benches/parser.rs @@ -1,27 +1,26 @@ #[cfg(unix)] mod unix { - use brush_parser::{parse_tokens, tokenize_str}; - use criterion::{black_box, Criterion}; + use brush_parser::{parse_tokens, uncached_tokenize_str, Token}; + use criterion::Criterion; - fn parse_script(contents: &str) -> brush_parser::ast::Program { - let tokens = tokenize_str(contents).unwrap(); + fn tokenize(content: &str) -> Vec { + uncached_tokenize_str(content, &brush_parser::TokenizerOptions::default()).unwrap() + } + + fn parse(tokens: &Vec) -> brush_parser::ast::Program { parse_tokens( - &tokens, + tokens, &brush_parser::ParserOptions::default(), &brush_parser::SourceInfo::default(), ) .unwrap() } - fn parse_sample_script() -> brush_parser::ast::Program { - let input = r#" - for f in A B C; do - echo "${f@L}" >&2 - done - "#; - - parse_script(input) - } + const SAMPLE_SCRIPT: &str = r#" +for f in A B C; do + echo "${f@L}" >&2 +done +"#; fn benchmark_parsing_script(c: &mut Criterion, script_path: &std::path::Path) { let contents = std::fs::read_to_string(script_path).unwrap(); @@ -32,15 +31,18 @@ mod unix { script_path.file_name().unwrap().to_string_lossy() ) .as_str(), - |b| b.iter(|| black_box(parse_script(contents.as_str()))), + |b| b.iter(|| parse(&tokenize(contents.as_str()))), ); } pub(crate) fn criterion_benchmark(c: &mut Criterion) { - c.bench_function("parse_sample_script", |b| { - b.iter(|| black_box(parse_sample_script())) + c.bench_function("tokenize_sample_script", |b| { + b.iter(|| tokenize(SAMPLE_SCRIPT)) }); + let tokens = tokenize(SAMPLE_SCRIPT); + c.bench_function("parse_sample_script", |b| b.iter(|| parse(&tokens))); + const POSSIBLE_BASH_COMPLETION_SCRIPT_PATH: &str = "/usr/share/bash-completion/bash_completion"; let well_known_complicated_script = diff --git a/brush-parser/src/lib.rs b/brush-parser/src/lib.rs index 3f902291..304b7daa 100644 --- a/brush-parser/src/lib.rs +++ b/brush-parser/src/lib.rs @@ -16,5 +16,6 @@ mod tokenizer; pub use error::{ParseError, TestCommandParseError, WordParseError}; pub use parser::{parse_tokens, Parser, ParserOptions, SourceInfo}; pub use tokenizer::{ - tokenize_str, tokenize_str_with_options, unquote_str, SourcePosition, Token, TokenLocation, + tokenize_str, tokenize_str_with_options, uncached_tokenize_str, unquote_str, SourcePosition, + Token, TokenLocation, TokenizerOptions, }; diff --git a/brush-parser/src/tokenizer.rs b/brush-parser/src/tokenizer.rs index a62c1e75..db859e4d 100644 --- a/brush-parser/src/tokenizer.rs +++ b/brush-parser/src/tokenizer.rs @@ -483,26 +483,41 @@ pub fn tokenize_str_with_options( input: &str, options: &TokenizerOptions, ) -> Result, TokenizerError> { - cacheable_tokenize_str(input.to_owned(), options.to_owned()) + uncached_tokenize_string(input.to_owned(), *options) } -#[cached::proc_macro::cached(size = 64, result = true)] -fn cacheable_tokenize_str( +#[cached::proc_macro::cached(name = "TOKENIZE_CACHE", size = 64, result = true)] +fn uncached_tokenize_string( input: String, options: TokenizerOptions, +) -> Result, TokenizerError> { + uncached_tokenize_str(input.as_str(), &options) +} + +/// Break the given input shell script string into tokens, returning the tokens. +/// No caching is performed. +/// +/// # Arguments +/// +/// * `input` - The shell script to tokenize. +pub fn uncached_tokenize_str( + input: &str, + options: &TokenizerOptions, ) -> Result, TokenizerError> { let mut reader = std::io::BufReader::new(input.as_bytes()); - let mut tokenizer = crate::tokenizer::Tokenizer::new(&mut reader, &options); + let mut tokenizer = crate::tokenizer::Tokenizer::new(&mut reader, options); let mut tokens = vec![]; loop { - let tokenize_result = tokenizer.next_token()?; - if let Some(token) = tokenize_result.token { - tokens.push(token); - } - - if matches!(tokenize_result.reason, TokenEndReason::EndOfInput) { - break; + match tokenizer.next_token()? { + TokenizeResult { + token: Some(token), .. + } => tokens.push(token), + TokenizeResult { + reason: TokenEndReason::EndOfInput, + .. + } => break, + _ => (), } }