diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0097bd40aa..c2cf119c87 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,7 @@ updates: time: "04:00" # UTC labels: - "domain: deps" + - "no-changelog" commit-message: prefix: "chore(deps)" open-pull-requests-limit: 10 diff --git a/.github/workflows/changelog.yaml b/.github/workflows/changelog.yaml new file mode 100644 index 0000000000..866224b793 --- /dev/null +++ b/.github/workflows/changelog.yaml @@ -0,0 +1,54 @@ +# Changelog +# +# Validates that a changelog entry was added. +# Runs on PRs when: +# - opened/re-opened +# - new commits pushed +# - label is added or removed +# Runs on merge queues, but just passes, because it is a required check. + +name: Changelog + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled] + # Due to merge queue requiring same status checks as PRs, must pass by default + merge_group: + types: [checks_requested] + +jobs: + validate-changelog: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + env: + PR_HAS_LABEL: ${{ contains( github.event.pull_request.labels.*.name, 'no-changelog') }} + steps: + - uses: actions/checkout@v4 + + - run: | + if [[ $PR_HAS_LABEL == 'true' ]] ; then + echo "'no-changelog' label detected." + exit 0 + fi + + # helper script needs to reference the master branch to compare against + git fetch origin main:refs/remotes/origin/main + + ./scripts/check_changelog_fragments.sh + + check-changelog: + name: Changelog + runs-on: ubuntu-latest + needs: validate-changelog + if: always() + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} + steps: + - name: exit + run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 654fd0038e..a17620be34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -544,7 +544,46 @@ # Changelog -## unreleased + This project uses [*towncrier*](https://towncrier.readthedocs.io/) for changelog generation. + + + +## [0.11.0 (2024-02-07)] + + +### New Features + +- Added `parse_etld` function for extracting eTLD and eTLD+1 (https://github.com/vectordotdev/vrl/pull/669) +- Added `encode_punycode` and `decode_punycode` functions (https://github.com/vectordotdev/vrl/pull/672) + +### Enhancements + +- Introduced a `redactor` option in `redact` function to enable the substitution of redacted content with either a custom string or a hash representation. (https://github.com/vectordotdev/vrl/pull/633) +- Add VRL function `get_timezone_name` to return the configured/resolved IANA timezone name. + + authors: klondikedragon (https://github.com/vectordotdev/vrl/pull/671) + +### Fixes + +- Fixed a bug in exporting paths containing more than one "coalesce" segment. (https://github.com/vectordotdev/vrl/pull/679) + + +## [0.10.0 (2024-01-24)] + + +### New Features + +- Introduced an unused expression checker. It's designed to detect and report unused expressions, + helping users to clean up and optimize their VRL scripts. Note that this checker will not catch everything, + but it does aim to eliminate false positives. For example, shadowed variables are not reported as unused. + (https://github.com/vectordotdev/vrl/pull/622) +- Add a `replace_with` function that is similar to `replace` but takes a closure instead of a + replacement string. (https://github.com/vectordotdev/vrl/pull/628) + +### Enhancements + +- Added the `alias_sources` parameter for `parse_groks` to read sources from files. (https://github.com/vectordotdev/vrl/pull/194) + ## `0.9.1` (2023-12-21) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3fa1301994..d49872b77c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,10 +15,10 @@ provide everything you need to get started. 4. Make your changes. 5. Add and/or update tests to cover your changes. 6. Run `./scripts/checks.sh` to run tests and other checks. -7. Add a one line summary of changes to the "unreleased" section of `CHANGELOG.md`. -8. [Submit the branch as a pull request][urls.submit_pr] to the repo. A team member should +7. [Submit the branch as a pull request][urls.submit_pr] to the repo. A team member should comment and/or review your pull request. - +8. Add a changelog fragment (requires the PR number) to describe your changes which will + be included in the release changelog. See the [README.md](changelog.d/README.md) for details. [urls.existing_issues]: https://github.com/vectordotdev/vrl/issues [urls.new_issue]: https://github.com/vectordotdev/vrl/issues/new diff --git a/Cargo.lock b/Cargo.lock index f10b52fe7d..18c9ea1df1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,9 +31,9 @@ dependencies = [ [[package]] name = "afl" -version = "0.15.1" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca431c501c567ea9a4e52c2ebca418299b8201afc15e6ea47584733f0faf9780" +checksum = "1ff7c9e6d8b0f28402139fcbff21a22038212c94c44ecf90812ce92f384308f6" dependencies = [ "home", "libc", @@ -93,9 +93,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.1" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cd65a4b849ace0b7f6daeebcc1a1d111282227ca745458c61dbf670e52a597" +checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" dependencies = [ "anstyle", "anstyle-parse", @@ -131,19 +131,19 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0238ca56c96dfa37bdf7c373c8886dd591322500aceeeccdb2216fe06dc2f796" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "arbitrary" @@ -189,9 +189,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bit-set" @@ -270,7 +270,7 @@ dependencies = [ "proc-macro-crate 2.0.0", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", "syn_derive", ] @@ -405,9 +405,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" dependencies = [ "android-tzdata", "iana-time-zone", @@ -415,14 +415,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.0", + "windows-targets 0.52.0", ] [[package]] name = "chrono-tz" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" +checksum = "91d7b79e99bfaa0d47da0687c43aa3b7381938a62ad3a6498599039321f660b7" dependencies = [ "chrono", "chrono-tz-build", @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -507,9 +507,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -526,7 +526,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] @@ -577,7 +577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f6af96839c04974cf381e427792a99913ecf3f7bfb348f153dc8a8e5f9803ad" dependencies = [ "anyhow", - "base64 0.21.5", + "base64 0.21.7", "hex", "lazy_static", "num_enum", @@ -779,7 +779,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] @@ -845,12 +845,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dyn-clone" version = "1.0.16" @@ -905,23 +899,12 @@ checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" [[package]] name = "errno" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -936,6 +919,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -952,6 +941,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1086,16 +1081,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -1129,9 +1124,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.1", @@ -1187,9 +1182,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -1289,6 +1284,12 @@ version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1325,9 +1326,9 @@ checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lua-src" -version = "546.0.0" +version = "546.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cb00c1380f1b4b4928dd211c07301ffa34872a239e590bd3219d9e5b213face" +checksum = "2da0daa7eee611a4c30c8f5ee31af55266e26e573971ba9336d2993e2da129b2" dependencies = [ "cc", ] @@ -1408,9 +1409,9 @@ dependencies = [ [[package]] name = "mlua" -version = "0.9.2" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c81f8ac20188feb5461a73eabb22a34dd09d6d58513535eb587e46bff6ba250" +checksum = "d111deb18a9c9bd33e1541309f4742523bfab01d276bfa9a27519f6de9c11dc7" dependencies = [ "bstr", "mlua-sys", @@ -1421,9 +1422,9 @@ dependencies = [ [[package]] name = "mlua-sys" -version = "0.4.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc29228347d6bdc9e613dc95c69df2817f755434ee0f7f3b27b57755fe238b7f" +checksum = "a088ed0723df7567f569ba018c5d48c23c501f3878b190b04144dfa5ebfa8abc" dependencies = [ "cc", "cfg-if", @@ -1503,6 +1504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1533,7 +1535,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] @@ -1670,9 +1672,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", "thiserror", @@ -1681,9 +1683,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.5" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" dependencies = [ "pest", "pest_generator", @@ -1691,22 +1693,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.5" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] name = "pest_meta" -version = "2.7.5" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", @@ -1900,13 +1902,44 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.4.1", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.2", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "proptest-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf16337405ca084e9c78985114633b6827711d22b9e6ef6c6c0d665eb3f0b6e" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "psl" version = "2.1.50" @@ -1942,6 +1975,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quickcheck" version = "1.0.3" @@ -1955,9 +1994,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.31" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2004,6 +2043,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + [[package]] name = "raw-window-handle" version = "0.5.2" @@ -2050,6 +2098,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -2063,13 +2120,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.4", "regex-syntax 0.8.2", ] @@ -2084,9 +2141,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" dependencies = [ "aho-corasick", "memchr", @@ -2172,9 +2229,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" @@ -2201,15 +2258,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys 0.4.11", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2218,6 +2275,18 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "rustyline" version = "13.0.0" @@ -2281,29 +2350,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -2312,9 +2381,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -2403,24 +2472,23 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snafu" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +checksum = "d342c51730e54029130d7dc9fd735d28c4cd360f1368c01981d4f03ff207f096" dependencies = [ - "doc-comment", "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +checksum = "080c44971436b1af15d6f61ddd8b543995cf63ab8e677d46b00cc06f4ef267a0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.46", ] [[package]] @@ -2495,9 +2563,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ "proc-macro2", "quote", @@ -2513,7 +2581,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] @@ -2532,6 +2600,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.4.1", + "rustix 0.38.28", + "windows-sys 0.52.0", +] + [[package]] name = "term" version = "0.7.0" @@ -2545,31 +2626,31 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", ] [[package]] @@ -2638,9 +2719,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "toml" -version = "0.8.8" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", @@ -2649,9 +2730,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] @@ -2662,7 +2743,7 @@ version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -2673,7 +2754,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -2776,6 +2857,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.13" @@ -2850,9 +2937,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", "wasm-bindgen", @@ -2872,14 +2959,14 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vrl" -version = "0.9.1" +version = "0.11.0" dependencies = [ "aes", "ansi_term", "anyhow", "arbitrary", "base16", - "base64 0.21.5", + "base64 0.21.7", "bytes", "cbc", "cfb-mode", @@ -2906,9 +2993,11 @@ dependencies = [ "hex", "hmac", "hostname", - "indexmap 2.1.0", + "iana-time-zone", + "idna", + "indexmap 2.2.6", "indoc", - "itertools 0.12.0", + "itertools 0.12.1", "lalrpop", "lalrpop-util", "md-5", @@ -2925,6 +3014,8 @@ dependencies = [ "pest_derive", "prettydiff", "prettytable-rs", + "proptest", + "proptest-derive", "psl", "quickcheck", "quoted_printable", @@ -3006,6 +3097,15 @@ dependencies = [ "quote", ] +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.3.3" @@ -3043,7 +3143,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", "wasm-bindgen-shared", ] @@ -3065,7 +3165,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.46", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3112,7 +3212,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.25", + "rustix 0.38.28", "windows-sys 0.48.0", ] @@ -3148,12 +3248,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.52.0", ] [[package]] @@ -3174,6 +3274,15 @@ dependencies = [ "windows-targets 0.48.0", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -3204,6 +3313,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.0", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -3216,6 +3340,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -3228,6 +3358,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -3240,6 +3376,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -3252,6 +3394,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -3264,6 +3412,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -3276,6 +3430,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -3288,6 +3448,12 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index ad7b5905cd..a42bbd71b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vrl" -version = "0.9.1" +version = "0.11.0" authors = ["Vector Contributors "] edition = "2021" license = "MPL-2.0" @@ -44,12 +44,63 @@ cli = ["stdlib", "dep:serde_json", "dep:thiserror", "dep:clap", "dep:exitcode", test_framework = ["compiler", "dep:prettydiff", "dep:serde_json", "dep:ansi_term"] arbitrary = ["dep:quickcheck", "dep:arbitrary"] lua = ["dep:mlua"] +proptest = ["dep:proptest", "dep:proptest-derive"] # Internal testing utilities (used for benches) test = ["string_path"] # All stdlib functions -stdlib = ["compiler", "core", "datadog", "dep:aes", "dep:chacha20poly1305", "dep:crypto_secretbox", "dep:ctr", "dep:cbc", "dep:cfb-mode", "dep:ofb", "dep:base16", "dep:nom", "dep:strip-ansi-escapes", "dep:utf8-width", "dep:hex", "dep:seahash", "dep:syslog_loose", "dep:hostname", "dep:zstd", "dep:quoted_printable", "dep:once_cell", "dep:base64", "dep:uuid", "dep:percent-encoding", "dep:uaparser", "dep:rust_decimal", "dep:indexmap", "dep:flate2", "dep:charset", "dep:data-encoding", "dep:hmac", "dep:digest", "dep:sha-1", "dep:cidr-utils", "dep:sha-2", "dep:md-5", "dep:url", "dep:woothee", "dep:csv", "dep:roxmltree", "dep:rand", "dep:dns-lookup", "dep:sha-3", "dep:grok", "dep:community-id", "dep:snap", "dep:substring", "dep:psl"] +stdlib = [ + "compiler", + "core", + "datadog", + "dep:aes", + "dep:base16", + "dep:base64", + "dep:cbc", + "dep:cfb-mode", + "dep:chacha20poly1305", + "dep:charset", + "dep:cidr-utils", + "dep:community-id", + "dep:crypto_secretbox", + "dep:csv", + "dep:ctr", + "dep:data-encoding", + "dep:digest", + "dep:dns-lookup", + "dep:flate2", + "dep:grok", + "dep:hex", + "dep:hmac", + "dep:hostname", + "dep:idna", + "dep:indexmap", + "dep:md-5", + "dep:nom", + "dep:ofb", + "dep:once_cell", + "dep:percent-encoding", + "dep:psl", + "dep:quoted_printable", + "dep:rand", + "dep:roxmltree", + "dep:rust_decimal", + "dep:seahash", + "dep:sha-1", + "dep:sha-2", + "dep:sha-3", + "dep:snap", + "dep:strip-ansi-escapes", + "dep:substring", + "dep:syslog_loose", + "dep:uaparser", + "dep:url", + "dep:utf8-width", + "dep:uuid", + "dep:woothee", + "dep:zstd", + ] [dependencies] cfg-if = "1.0.0" @@ -62,10 +113,10 @@ base64 = { version = "0.21", optional = true } bytes = { version = "1.5.0", default-features = false, optional = true } charset = { version = "0.1.3", optional = true } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"], optional = true } -chrono-tz = { version = "0.8.4", default-features = false, optional = true } +chrono-tz = { version = "0.8.5", default-features = false, optional = true } cidr-utils = { version = "0.6", optional = true } csv = { version = "1.3", optional = true } -clap = { version = "4.4.10", features = ["derive"], optional = true } +clap = { version = "4.4.18", features = ["derive"], optional = true } codespan-reporting = {version = "0.11", optional = true } data-encoding = { version = "2.5.0", optional = true } digest = { version = "0.10", optional = true } @@ -74,11 +125,13 @@ exitcode = {version = "1", optional = true } flate2 = { version = "1.0.28", default-features = false, features = ["default"], optional = true } hex = { version = "0.4", optional = true } hmac = { version = "0.12.1", optional = true } -indexmap = { version = "~2.1.0", default-features = false, features = ["std"], optional = true} +idna = { version = "0.5", optional = true } +iana-time-zone = "0.1.59" +indexmap = { version = "~2.2.2", default-features = false, features = ["std"], optional = true} indoc = {version = "2.0.4", optional = true } -itertools = { version = "0.12.0", default-features = false, optional = true } +itertools = { version = "0.12.1", default-features = false, optional = true } lalrpop-util = { version = "0.20", optional = true } -mlua = { version = "0.9.2", default-features = false, features = ["lua54", "send", "vendored"], optional = true} +mlua = { version = "0.9.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true} nom = { version = "7.1.3", default-features = false, features = ["std"], optional = true } once_cell = { version = "1.19", default-features = false, features = ["std"], optional = true } ordered-float = { version = "4", default-features = false, optional = true } @@ -86,8 +139,10 @@ md-5 = { version = "0.10", optional = true } paste = { version = "1", default-features = false, optional = true } peeking_take_while = { version = "1.0.0", default-features = false, optional = true } percent-encoding = { version = "2.3", optional = true } -pest = { version = "2.7.5", default-features = false, optional = true, features = ["std"] } -pest_derive = { version = "2.7.5", default-features = false, optional = true, features = ["std"] } +pest = { version = "2.7.7", default-features = false, optional = true, features = ["std"] } +pest_derive = { version = "2.7.7", default-features = false, optional = true, features = ["std"] } +proptest = { version = "1.4", optional = true } +proptest-derive = { version = "0.4", optional = true } prettydiff = {version = "0.6", default-features = false, optional = true } prettytable-rs = { version = "0.10", default-features = false, optional = true } quickcheck = { version = "1.0.3", optional = true } @@ -100,7 +155,7 @@ rustyline = { version = "13", default-features = false, optional = true } rust_decimal = { version = "1", optional = true } seahash = { version = "4.1.0", optional = true } serde = { version = "1", features = ["derive"], optional = true } -serde_json = { version = "1.0.108", default-features = false, optional = true, features = ["std", "raw_value"] } +serde_json = { version = "1.0.113", default-features = false, optional = true, features = ["std", "raw_value"] } sha-1 = { version = "0.10", optional = true } sha-2 = { package = "sha2", version = "0.10", optional = true } sha-3 = { package = "sha3", version = "0.10", optional = true } @@ -114,7 +169,7 @@ tracing = { version = "0.1.40", default-features = false, optional = true } uaparser = { version = "0.6.1", default-features = false, optional = true } utf8-width = { version = "0.1.7", optional = true } url = { version = "2", optional = true } -snafu = { version = "0.7", optional = true } +snafu = { version = "0.8", optional = true } webbrowser = { version = "0.8", default-features = false, optional = true } woothee = { version = "0.13.0", optional = true } community-id = { version = "0.2.2", optional = true} @@ -151,11 +206,13 @@ chrono-tz = "0.8" serde_json = "1" indoc = "2.0.4" tracing-test = { version = "0.2", default-features = false } -toml = { version = "0.8.8", default-features = false } -mlua = { version = "0.9.2", default-features = false, features = ["lua54", "send", "vendored"]} +toml = { version = "0.8.10", default-features = false } +mlua = { version = "0.9.5", default-features = false, features = ["lua54", "send", "vendored"]} quickcheck = { version = "1.0.3"} regex = { version = "1", default-features = false, features = ["std", "perf", "unicode"] } paste = { version = "1", default-features = false } +proptest = { version = "1.4" } +proptest-derive = { version = "0.4" } [build-dependencies] lalrpop = { version = "0.20", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 2e72afe8fa..6c6a995572 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -60,14 +60,12 @@ digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Develop dirs-next,https://github.com/xdg-rs/dirs,MIT OR Apache-2.0,The @xdg-rs members dirs-sys-next,https://github.com/xdg-rs/dirs/tree/master/dirs-sys,MIT OR Apache-2.0,The @xdg-rs members dns-lookup,https://github.com/keeperofdakeys/dns-lookup,MIT OR Apache-2.0,Josh Driver -doc-comment,https://github.com/GuillaumeGomez/doc-comment,MIT,Guillaume Gomez dyn-clone,https://github.com/dtolnay/dyn-clone,MIT OR Apache-2.0,David Tolnay either,https://github.com/bluss/either,MIT OR Apache-2.0,bluss encode_unicode,https://github.com/tormol/encode_unicode,Apache-2.0 OR MIT,Torbjørn Birch Moltu encoding_rs,https://github.com/hsivonen/encoding_rs,(Apache-2.0 OR MIT) AND BSD-3-Clause,Henri Sivonen equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,Chris Wong -errno-dragonfly,https://github.com/mneumann/errno-dragonfly-rs,MIT,Michael Neumann error-code,https://github.com/DoumanAsh/error-code,BSL-1.0,Douman exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " @@ -85,6 +83,7 @@ hostname,https://github.com/svartalf/hostname,MIT,"fengcen , René Kijewski , Ryan Lopopolo " iana-time-zone-haiku,https://github.com/strawlab/iana-time-zone,MIT OR Apache-2.0,René Kijewski indexmap,https://github.com/bluss/indexmap,Apache-2.0 OR MIT,The indexmap Authors +indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap Authors indoc,https://github.com/dtolnay/indoc,MIT OR Apache-2.0,David Tolnay inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers io-lifetimes,https://github.com/sunfishcode/io-lifetimes,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman @@ -98,6 +97,7 @@ keccak,https://github.com/RustCrypto/sponges/tree/master/keccak,Apache-2.0 OR MI lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers +libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov , Andrew Paseltiner " linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers @@ -133,6 +133,8 @@ prettytable-rs,https://github.com/phsym/prettytable-rs,BSD-3-Clause,Pierre-Henri proc-macro-crate,https://github.com/bkchr/proc-macro-crate,MIT OR Apache-2.0,Bastian Köcher proc-macro-error,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " +psl,https://github.com/addr-rs/psl,MIT OR Apache-2.0,rushmorem +psl-types,https://github.com/addr-rs/psl-types,MIT OR Apache-2.0,rushmorem ptr_meta,https://github.com/djkoloski/ptr_meta,MIT,David Koloski quote,https://github.com/dtolnay/quote,MIT OR Apache-2.0,David Tolnay quoted_printable,https://github.com/staktrace/quoted-printable,0BSD,Kartikaya Gupta @@ -208,7 +210,16 @@ web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm winapi,https://github.com/retep998/winapi-rs,MIT OR Apache-2.0,Peter Atashian winapi-util,https://github.com/BurntSushi/winapi-util,Unlicense OR MIT,Andrew Gallant -windows,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_i686_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto wyz,https://github.com/myrrlyn/wyz,MIT,myrrlyn diff --git a/benches/stdlib.rs b/benches/stdlib.rs index 5812ae6928..d5276df871 100644 --- a/benches/stdlib.rs +++ b/benches/stdlib.rs @@ -25,6 +25,7 @@ criterion_group!( decode_base16, decode_base64, decode_percent, + decode_punycode, decrypt, // TODO: Cannot pass a Path to bench_function //del, @@ -35,6 +36,7 @@ criterion_group!( encode_json, encode_logfmt, encode_percent, + encode_punycode, encrypt, ends_with, // TODO: Cannot pass a Path to bench_function @@ -49,6 +51,7 @@ criterion_group!( get, get_env_var, get_hostname, + get_timezone_name, includes, int, ip_aton, @@ -94,6 +97,7 @@ criterion_group!( parse_common_log, parse_csv, parse_duration, + parse_etld, parse_glog, parse_grok, parse_groks, @@ -299,6 +303,20 @@ bench_function! { } } +bench_function! { + decode_punycode => vrl::stdlib::DecodePunycode; + + encoded { + args: func_args![value: "www.xn--caf-dma.com"], + want: Ok("www.café.com"), + } + + non_encoded { + args: func_args![value: "www.cafe.com"], + want: Ok("www.cafe.com"), + } +} + bench_function! { decode_mime_q => vrl::stdlib::DecodeMimeQ; @@ -441,6 +459,20 @@ bench_function! { } } +bench_function! { + encode_punycode => vrl::stdlib::EncodePunycode; + + idn { + args: func_args![value: "www.CAFé.com"], + want: Ok("www.xn--caf-dma.com"), + } + + ascii { + args: func_args![value: "www.cafe.com"], + want: Ok("www.cafe.com"), + } +} + bench_function! { ends_with => vrl::stdlib::EndsWith; @@ -578,6 +610,15 @@ bench_function! { } } +bench_function! { + get_timezone_name => vrl::stdlib::GetTimezoneName; + + get { + args: func_args![], + want: Ok(vrl::stdlib::get_name_for_timezone(&vrl::compiler::TimeZone::Named(chrono_tz::Tz::UTC))), + } +} + bench_function! { includes => vrl::stdlib::Includes; @@ -1560,6 +1601,19 @@ bench_function! { } } +bench_function! { + parse_etld => vrl::stdlib::ParseEtld; + + literal { + args: func_args![value: "vector.dev"], + want: Ok(Value::from(btreemap! { + "etld" => "dev", + "etld_plus" => "dev", + "known_suffix" => true + })) + } +} + bench_function! { parse_glog => vrl::stdlib::ParseGlog; diff --git a/changelog.d/README.md b/changelog.d/README.md new file mode 100644 index 0000000000..bf99cff382 --- /dev/null +++ b/changelog.d/README.md @@ -0,0 +1,75 @@ +## Overview + +This directory contains changelog "fragments" that are collected during a release to +generate the project's changelog. + +The conventions used for this changelog logic follow [towncrier](https://towncrier.readthedocs.io/en/stable/markdown.html). + +The changelog fragments are located in `changelog.d/`. + +## Process + +Fragments for un-released changes are placed in the root of this directory during PRs. + +During a release, `scripts/generate_release_changelog.sh` is run in order to automatically +generate the changes to the CHANGELOG.md file. As part of the script execution, the +changelog fragment files that are being released, are removed from the repo. + +### Pull Requests + +By default, PRs are required to add at least one entry to this directory. +This is enforced during CI. + +To mark a PR as not requiring changelog notes, add the label 'no-changelog'. + +To run the same check that is run in CI to validate that your changelog fragments have +the correct syntax, commit the fragment additions and then run `./scripts/check_changelog_fragments.sh` + +The format for fragments is: `..md` + +### Fragment conventions + +When fragments used to generate the updated changelog, the content of the fragment file is +rendered as an item in a bulleted list under the "type" of fragment. + +The contents of the file must be valid markdown. + +Filename rules: + +- The first segment (pr_number) must match the GitHub PR the change is introduced in. +- The type must be one of the valid types in [Fragment types](#types) +- Only the two period delimiters can be used. +- The file must be markdown. + +#### Fragment types {#types} + +- breaking: A change that is incompatible with prior versions which requires users to make adjustments. +- security: A change that is has implications for security. +- deprecation: A change that is introducing a deprecation. +- feature: A change that is introducing a new feature. +- enhancement: A change that is enhancing existing functionality in a user perceivable way. +- fix: A change that is fixing a bug. + +#### Fragment contents + +When fragments are rendered in the changelog, each fragment becomes an item in a markdown list. +For this reason, when creating the content in a fragment, the format must be renderable as a markdown list. + +As an example, separating content with markdown header syntax should be avoided, as that will render +as a heading in the main changelog and not the list. Instead, separate content with newlines. + +### Breaking changes + +When using the type 'breaking' to add notes for a breaking change, these should be more verbose than +other entries typically. It should include all details that would be relevant for the user to need +to handle upgrading to the breaking change. + +## Example + +Here is an example of a changelog fragment that adds a breaking change explanation. + + $ cat changelog.d/42_very_good_words.breaking.md + This change is so great. It's such a great change that this sentence + explaining the change has to span multiple lines of text. + + It even necessitates a line break. It is a breaking change after all. diff --git a/changelog.toml b/changelog.toml new file mode 100644 index 0000000000..bc447e453f --- /dev/null +++ b/changelog.toml @@ -0,0 +1,57 @@ +# Configuration for `towncrier` used during release process to auto-generate changelog. + +[tool.towncrier] +directory = "changelog.d" +filename = "CHANGELOG.md" +start_string = "\n" +underlines = ["", "", ""] +title_format = "## [{version} ({project_date})]" +issue_format = "https://github.com/vectordotdev/vrl/pull/{issue}" + +# The following configurations specify which fragment "types" are +# allowed. +# +# If a change applies to more than one type, select the one it most +# applies to. Or, if applicable, multiple changelog fragments can be +# added for one PR. For example, if a PR includes a breaking change +# around some feature, but also fixes a bug in the same part of the +# code but is tangential to the breaking change, a separate +# fragment can be added to call out the fix. + +# A change that is incompatible with prior versions which +# requires users to make adjustments. +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking Changes & Upgrade Guide" +showcontent = true + +# A change that has implications for security. +[[tool.towncrier.type]] +directory = "security" +name = "Security" +showcontent = true + +# A change that is introducing a deprecation. +[[tool.towncrier.type]] +directory = "deprecation" +name = "Deprecations" +showcontent = true + +# A change that is introducing a new feature. +[[tool.towncrier.type]] +directory = "feature" +name = "New Features" +showcontent = true + +# A change that is enhancing existing functionality in a user +# perceivable way. +[[tool.towncrier.type]] +directory = "enhancement" +name = "Enhancements" +showcontent = true + +# A change that is fixing a bug. +[[tool.towncrier.type]] +directory = "fix" +name = "Fixes" +showcontent = true diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 70f3fce1d8..86280ef355 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -11,5 +11,5 @@ name = "vrl" path = "src/main.rs" [dependencies] -clap = { version = "4.4.10", features = ["derive"] } +clap = { version = "4.4.18", features = ["derive"] } vrl = { path = "../../", features = ["cli"] } diff --git a/lib/fuzz/Cargo.toml b/lib/fuzz/Cargo.toml index f3725cec77..870eb9dc8b 100644 --- a/lib/fuzz/Cargo.toml +++ b/lib/fuzz/Cargo.toml @@ -11,5 +11,5 @@ name = "fuzz" path = "src/main.rs" [dependencies] -afl = "0.15.1" +afl = "0.15.3" vrl = { path = "../../" } diff --git a/lib/tests/tests/diagnostics/truncate_deprecated_argument.vrl b/lib/tests/tests/diagnostics/truncate_deprecated_argument.vrl index 186bf8bad5..e68b0650d5 100644 --- a/lib/tests/tests/diagnostics/truncate_deprecated_argument.vrl +++ b/lib/tests/tests/diagnostics/truncate_deprecated_argument.vrl @@ -1,3 +1,4 @@ +# DIAGNOSTICS # result: # # warning[E801]: the `ellipsis` argument is deprecated since v0.7.0 diff --git a/lib/tests/tests/diagnostics/unusued/top_level.vrl b/lib/tests/tests/diagnostics/unusued/top_level.vrl new file mode 100644 index 0000000000..46177ea4a8 --- /dev/null +++ b/lib/tests/tests/diagnostics/unusued/top_level.vrl @@ -0,0 +1,25 @@ +# DIAGNOSTICS +# result: +# +# warning[E900]: unused literal `"foo"` +# ┌─ :2:1 +# │ +# 2 │ "foo" +# │ ^^^^^ help: use the result of this expression or remove it +# │ +# = this expression has no side-effects +# = see language documentation at https://vrl.dev +# = try your code in the VRL REPL, learn more at https://vrl.dev/examples +# warning[E900]: unused variable `x` +# ┌─ :4:1 +# │ +# 4 │ x = "bar" +# │ ^ help: use the result of this expression or remove it +# │ +# = this expression has no side-effects +# = see language documentation at https://vrl.dev +# = try your code in the VRL REPL, learn more at https://vrl.dev/examples + +"foo" + +x = "bar" diff --git a/lib/tests/tests/expressions/arithmetic/addition/invalid.vrl b/lib/tests/tests/expressions/arithmetic/addition/invalid.vrl index 62207d6469..85d86ff229 100644 --- a/lib/tests/tests/expressions/arithmetic/addition/invalid.vrl +++ b/lib/tests/tests/expressions/arithmetic/addition/invalid.vrl @@ -92,113 +92,113 @@ # "can't add type timestamp to timestamp" # ] -ok, err1 = 1 + "foo" -ok, err2 = 1 + true -ok, err3 = 1 + false -ok, err4 = 1 + null -ok, err5 = 1 + [] -ok, err6 = 1 + {} -ok, err7 = 1 + r'foo' -ok, err8 = 1 + now() +_ok, err1 = 1 + "foo" +_ok, err2 = 1 + true +_ok, err3 = 1 + false +_ok, err4 = 1 + null +_ok, err5 = 1 + [] +_ok, err6 = 1 + {} +_ok, err7 = 1 + r'foo' +_ok, err8 = 1 + now() integer_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = 1.0 + "foo" -ok, err2 = 1.0 + true -ok, err3 = 1.0 + false -ok, err4 = 1.0 + null -ok, err5 = 1.0 + [] -ok, err6 = 1.0 + {} -ok, err7 = 1.0 + r'foo' -ok, err8 = 1.0 + now() +_ok, err1 = 1.0 + "foo" +_ok, err2 = 1.0 + true +_ok, err3 = 1.0 + false +_ok, err4 = 1.0 + null +_ok, err5 = 1.0 + [] +_ok, err6 = 1.0 + {} +_ok, err7 = 1.0 + r'foo' +_ok, err8 = 1.0 + now() float_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = "foo" + 1 -ok, err2 = "foo" + 1.0 -ok, err3 = "foo" + true -ok, err4 = "foo" + false -ok, err5 = "foo" + [] -ok, err6 = "foo" + {} -ok, err7 = "foo" + r'foo' -ok, err8 = "foo" + now() +_ok, err1 = "foo" + 1 +_ok, err2 = "foo" + 1.0 +_ok, err3 = "foo" + true +_ok, err4 = "foo" + false +_ok, err5 = "foo" + [] +_ok, err6 = "foo" + {} +_ok, err7 = "foo" + r'foo' +_ok, err8 = "foo" + now() string_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = true + 1 -ok, err2 = true + 1.0 -ok, err3 = true + "foo" -ok, err4 = true + true -ok, err5 = true + false -ok, err6 = true + null -ok, err7 = true + [] -ok, err8 = true + {} -ok, err9 = true + r'foo' -ok, err10 = true + now() +_ok, err1 = true + 1 +_ok, err2 = true + 1.0 +_ok, err3 = true + "foo" +_ok, err4 = true + true +_ok, err5 = true + false +_ok, err6 = true + null +_ok, err7 = true + [] +_ok, err8 = true + {} +_ok, err9 = true + r'foo' +_ok, err10 = true + now() bool_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = null + 1 -ok, err2 = null + 1.0 -ok, err3 = null + true -ok, err4 = null + false -ok, err5 = null + null -ok, err6 = null + [] -ok, err7 = null + {} -ok, err8 = null + r'foo' -ok, err9 = null + now() +_ok, err1 = null + 1 +_ok, err2 = null + 1.0 +_ok, err3 = null + true +_ok, err4 = null + false +_ok, err5 = null + null +_ok, err6 = null + [] +_ok, err7 = null + {} +_ok, err8 = null + r'foo' +_ok, err9 = null + now() null_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9] -ok, err1 = [] + "foo" -ok, err2 = [] + 1 -ok, err3 = [] + 1.0 -ok, err4 = [] + true -ok, err5 = [] + false -ok, err6 = [] + null -ok, err7 = [] + [] -ok, err8 = [] + {} -ok, err9 = [] + r'foo' -ok, err10 = [] + now() +_ok, err1 = [] + "foo" +_ok, err2 = [] + 1 +_ok, err3 = [] + 1.0 +_ok, err4 = [] + true +_ok, err5 = [] + false +_ok, err6 = [] + null +_ok, err7 = [] + [] +_ok, err8 = [] + {} +_ok, err9 = [] + r'foo' +_ok, err10 = [] + now() array_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = {} + "foo" -ok, err2 = {} + 1 -ok, err3 = {} + 1.0 -ok, err4 = {} + true -ok, err5 = {} + false -ok, err6 = {} + null -ok, err7 = {} + [] -ok, err8 = {} + {} -ok, err9 = {} + r'foo' -ok, err10 = {} + now() +_ok, err1 = {} + "foo" +_ok, err2 = {} + 1 +_ok, err3 = {} + 1.0 +_ok, err4 = {} + true +_ok, err5 = {} + false +_ok, err6 = {} + null +_ok, err7 = {} + [] +_ok, err8 = {} + {} +_ok, err9 = {} + r'foo' +_ok, err10 = {} + now() object_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = r'' + "foo" -ok, err2 = r'' + 1 -ok, err3 = r'' + 1.0 -ok, err4 = r'' + true -ok, err5 = r'' + false -ok, err6 = r'' + null -ok, err7 = r'' + [] -ok, err8 = r'' + {} -ok, err9 = r'' + r'foo' -ok, err10 = r'' + now() +_ok, err1 = r'' + "foo" +_ok, err2 = r'' + 1 +_ok, err3 = r'' + 1.0 +_ok, err4 = r'' + true +_ok, err5 = r'' + false +_ok, err6 = r'' + null +_ok, err7 = r'' + [] +_ok, err8 = r'' + {} +_ok, err9 = r'' + r'foo' +_ok, err10 = r'' + now() regex_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = from_unix_timestamp(0) + "foo" -ok, err2 = from_unix_timestamp(0) + 1 -ok, err3 = from_unix_timestamp(0) + 1.0 -ok, err4 = from_unix_timestamp(0) + true -ok, err5 = from_unix_timestamp(0) + false -ok, err6 = from_unix_timestamp(0) + null -ok, err7 = from_unix_timestamp(0) + [] -ok, err8 = from_unix_timestamp(0) + {} -ok, err9 = from_unix_timestamp(0) + r'foo' -ok, err10 = from_unix_timestamp(0) + now() +_ok, err1 = from_unix_timestamp(0) + "foo" +_ok, err2 = from_unix_timestamp(0) + 1 +_ok, err3 = from_unix_timestamp(0) + 1.0 +_ok, err4 = from_unix_timestamp(0) + true +_ok, err5 = from_unix_timestamp(0) + false +_ok, err6 = from_unix_timestamp(0) + null +_ok, err7 = from_unix_timestamp(0) + [] +_ok, err8 = from_unix_timestamp(0) + {} +_ok, err9 = from_unix_timestamp(0) + r'foo' +_ok, err10 = from_unix_timestamp(0) + now() timestamp_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] diff --git a/lib/tests/tests/expressions/arithmetic/division/invalid.vrl b/lib/tests/tests/expressions/arithmetic/division/invalid.vrl index 84646c2cf9..d772e652a8 100644 --- a/lib/tests/tests/expressions/arithmetic/division/invalid.vrl +++ b/lib/tests/tests/expressions/arithmetic/division/invalid.vrl @@ -93,114 +93,114 @@ # "can't divide type timestamp by timestamp" # ] -ok, err1 = 1 / true -ok, err2 = 1 / false -ok, err3 = 1 / null -ok, err4 = 1 / [] -ok, err5 = 1 / {} -ok, err6 = 1 / r'foo' -ok, err7 = 1 / now() +_ok, err1 = 1 / true +_ok, err2 = 1 / false +_ok, err3 = 1 / null +_ok, err4 = 1 / [] +_ok, err5 = 1 / {} +_ok, err6 = 1 / r'foo' +_ok, err7 = 1 / now() integer_errs = [err1, err2, err3, err4, err5, err6, err7] -ok, err1 = 1.0 / "foo" -ok, err2 = 1.0 / true -ok, err3 = 1.0 / false -ok, err4 = 1.0 / null -ok, err5 = 1.0 / [] -ok, err6 = 1.0 / {} -ok, err7 = 1.0 / r'foo' -ok, err8 = 1.0 / now() +_ok, err1 = 1.0 / "foo" +_ok, err2 = 1.0 / true +_ok, err3 = 1.0 / false +_ok, err4 = 1.0 / null +_ok, err5 = 1.0 / [] +_ok, err6 = 1.0 / {} +_ok, err7 = 1.0 / r'foo' +_ok, err8 = 1.0 / now() float_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = "foo" / "foo" -ok, err2 = "foo" / 1.0 -ok, err3 = "foo" / true -ok, err4 = "foo" / false -ok, err5 = "foo" / null -ok, err6 = "foo" / [] -ok, err7 = "foo" / {} -ok, err8 = "foo" / r'foo' -ok, err9 = "foo" / now() +_ok, err1 = "foo" / "foo" +_ok, err2 = "foo" / 1.0 +_ok, err3 = "foo" / true +_ok, err4 = "foo" / false +_ok, err5 = "foo" / null +_ok, err6 = "foo" / [] +_ok, err7 = "foo" / {} +_ok, err8 = "foo" / r'foo' +_ok, err9 = "foo" / now() string_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9] -ok, err1 = true / "foo" -ok, err2 = true / 1 -ok, err3 = true / 1.0 -ok, err4 = true / true -ok, err5 = true / false -ok, err6 = true / null -ok, err7 = true / [] -ok, err8 = true / {} -ok, err9 = true / r'foo' -ok, err10 = true / now() +_ok, err1 = true / "foo" +_ok, err2 = true / 1 +_ok, err3 = true / 1.0 +_ok, err4 = true / true +_ok, err5 = true / false +_ok, err6 = true / null +_ok, err7 = true / [] +_ok, err8 = true / {} +_ok, err9 = true / r'foo' +_ok, err10 = true / now() bool_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = null / "foo" -ok, err2 = null / 1 -ok, err3 = null / 1.0 -ok, err4 = null / true -ok, err5 = null / false -ok, err6 = null / null -ok, err7 = null / [] -ok, err8 = null / {} -ok, err9 = null / r'foo' -ok, err10 = null / now() +_ok, err1 = null / "foo" +_ok, err2 = null / 1 +_ok, err3 = null / 1.0 +_ok, err4 = null / true +_ok, err5 = null / false +_ok, err6 = null / null +_ok, err7 = null / [] +_ok, err8 = null / {} +_ok, err9 = null / r'foo' +_ok, err10 = null / now() null_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = [] / "foo" -ok, err2 = [] / 1 -ok, err3 = [] / 1.0 -ok, err4 = [] / true -ok, err5 = [] / false -ok, err6 = [] / null -ok, err7 = [] / [] -ok, err8 = [] / {} -ok, err9 = [] / r'foo' -ok, err10 = [] / now() +_ok, err1 = [] / "foo" +_ok, err2 = [] / 1 +_ok, err3 = [] / 1.0 +_ok, err4 = [] / true +_ok, err5 = [] / false +_ok, err6 = [] / null +_ok, err7 = [] / [] +_ok, err8 = [] / {} +_ok, err9 = [] / r'foo' +_ok, err10 = [] / now() array_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = {} / "foo" -ok, err2 = {} / 1 -ok, err3 = {} / 1.0 -ok, err4 = {} / true -ok, err5 = {} / false -ok, err6 = {} / null -ok, err7 = {} / [] -ok, err8 = {} / {} -ok, err9 = {} / r'foo' -ok, err10 = {} / now() +_ok, err1 = {} / "foo" +_ok, err2 = {} / 1 +_ok, err3 = {} / 1.0 +_ok, err4 = {} / true +_ok, err5 = {} / false +_ok, err6 = {} / null +_ok, err7 = {} / [] +_ok, err8 = {} / {} +_ok, err9 = {} / r'foo' +_ok, err10 = {} / now() object_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = r'' / "foo" -ok, err2 = r'' / 1 -ok, err3 = r'' / 1.0 -ok, err4 = r'' / true -ok, err5 = r'' / false -ok, err6 = r'' / null -ok, err7 = r'' / [] -ok, err8 = r'' / {} -ok, err9 = r'' / r'foo' -ok, err10 = r'' / now() +_ok, err1 = r'' / "foo" +_ok, err2 = r'' / 1 +_ok, err3 = r'' / 1.0 +_ok, err4 = r'' / true +_ok, err5 = r'' / false +_ok, err6 = r'' / null +_ok, err7 = r'' / [] +_ok, err8 = r'' / {} +_ok, err9 = r'' / r'foo' +_ok, err10 = r'' / now() regex_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = from_unix_timestamp(0) / "foo" -ok, err2 = from_unix_timestamp(0) / 1 -ok, err3 = from_unix_timestamp(0) / 1.0 -ok, err4 = from_unix_timestamp(0) / true -ok, err5 = from_unix_timestamp(0) / false -ok, err6 = from_unix_timestamp(0) / null -ok, err7 = from_unix_timestamp(0) / [] -ok, err8 = from_unix_timestamp(0) / {} -ok, err9 = from_unix_timestamp(0) / r'foo' -ok, err10 = from_unix_timestamp(0) / now() +_ok, err1 = from_unix_timestamp(0) / "foo" +_ok, err2 = from_unix_timestamp(0) / 1 +_ok, err3 = from_unix_timestamp(0) / 1.0 +_ok, err4 = from_unix_timestamp(0) / true +_ok, err5 = from_unix_timestamp(0) / false +_ok, err6 = from_unix_timestamp(0) / null +_ok, err7 = from_unix_timestamp(0) / [] +_ok, err8 = from_unix_timestamp(0) / {} +_ok, err9 = from_unix_timestamp(0) / r'foo' +_ok, err10 = from_unix_timestamp(0) / now() timestamp_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] diff --git a/lib/tests/tests/expressions/arithmetic/merge/invalid.vrl b/lib/tests/tests/expressions/arithmetic/merge/invalid.vrl index 8831673a71..e8a16ed16b 100644 --- a/lib/tests/tests/expressions/arithmetic/merge/invalid.vrl +++ b/lib/tests/tests/expressions/arithmetic/merge/invalid.vrl @@ -3,6 +3,6 @@ # only objects can be merged # only objects can be merged -ok = { "foo": 43 } | 34 -ok = { "foo": 43 } | [ 1, 2, 3 ] -ok = { "foo": 43 } | "ook" +_ok = { "foo": 43 } | 34 +_ok = { "foo": 43 } | [ 1, 2, 3 ] +_ok = { "foo": 43 } | "ook" diff --git a/lib/tests/tests/expressions/arithmetic/multiplication/invalid.vrl b/lib/tests/tests/expressions/arithmetic/multiplication/invalid.vrl index c41880b1da..aeaeb573c8 100644 --- a/lib/tests/tests/expressions/arithmetic/multiplication/invalid.vrl +++ b/lib/tests/tests/expressions/arithmetic/multiplication/invalid.vrl @@ -93,114 +93,114 @@ # "can't multiply type timestamp by timestamp" # ] -ok, err1 = 1 * true -ok, err2 = 1 * false -ok, err3 = 1 * null -ok, err4 = 1 * [] -ok, err5 = 1 * {} -ok, err6 = 1 * r'foo' -ok, err7 = 1 * now() +_ok, err1 = 1 * true +_ok, err2 = 1 * false +_ok, err3 = 1 * null +_ok, err4 = 1 * [] +_ok, err5 = 1 * {} +_ok, err6 = 1 * r'foo' +_ok, err7 = 1 * now() integer_errs = [err1, err2, err3, err4, err5, err6, err7] -ok, err1 = 1.0 * "foo" -ok, err2 = 1.0 * true -ok, err3 = 1.0 * false -ok, err4 = 1.0 * null -ok, err5 = 1.0 * [] -ok, err6 = 1.0 * {} -ok, err7 = 1.0 * r'foo' -ok, err8 = 1.0 * now() +_ok, err1 = 1.0 * "foo" +_ok, err2 = 1.0 * true +_ok, err3 = 1.0 * false +_ok, err4 = 1.0 * null +_ok, err5 = 1.0 * [] +_ok, err6 = 1.0 * {} +_ok, err7 = 1.0 * r'foo' +_ok, err8 = 1.0 * now() float_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = "foo" * "foo" -ok, err2 = "foo" * 1.0 -ok, err3 = "foo" * true -ok, err4 = "foo" * false -ok, err5 = "foo" * null -ok, err6 = "foo" * [] -ok, err7 = "foo" * {} -ok, err8 = "foo" * r'foo' -ok, err9 = "foo" * now() +_ok, err1 = "foo" * "foo" +_ok, err2 = "foo" * 1.0 +_ok, err3 = "foo" * true +_ok, err4 = "foo" * false +_ok, err5 = "foo" * null +_ok, err6 = "foo" * [] +_ok, err7 = "foo" * {} +_ok, err8 = "foo" * r'foo' +_ok, err9 = "foo" * now() string_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9] -ok, err1 = true * "foo" -ok, err2 = true * 1 -ok, err3 = true * 1.0 -ok, err4 = true * true -ok, err5 = true * false -ok, err6 = true * null -ok, err7 = true * [] -ok, err8 = true * {} -ok, err9 = true * r'foo' -ok, err10 = true * now() +_ok, err1 = true * "foo" +_ok, err2 = true * 1 +_ok, err3 = true * 1.0 +_ok, err4 = true * true +_ok, err5 = true * false +_ok, err6 = true * null +_ok, err7 = true * [] +_ok, err8 = true * {} +_ok, err9 = true * r'foo' +_ok, err10 = true * now() bool_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = null * "foo" -ok, err2 = null * 1 -ok, err3 = null * 1.0 -ok, err4 = null * true -ok, err5 = null * false -ok, err6 = null * null -ok, err7 = null * [] -ok, err8 = null * {} -ok, err9 = null * r'foo' -ok, err10 = null * now() +_ok, err1 = null * "foo" +_ok, err2 = null * 1 +_ok, err3 = null * 1.0 +_ok, err4 = null * true +_ok, err5 = null * false +_ok, err6 = null * null +_ok, err7 = null * [] +_ok, err8 = null * {} +_ok, err9 = null * r'foo' +_ok, err10 = null * now() null_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = [] * "foo" -ok, err2 = [] * 1 -ok, err3 = [] * 1.0 -ok, err4 = [] * true -ok, err5 = [] * false -ok, err6 = [] * null -ok, err7 = [] * [] -ok, err8 = [] * {} -ok, err9 = [] * r'foo' -ok, err10 = [] * now() +_ok, err1 = [] * "foo" +_ok, err2 = [] * 1 +_ok, err3 = [] * 1.0 +_ok, err4 = [] * true +_ok, err5 = [] * false +_ok, err6 = [] * null +_ok, err7 = [] * [] +_ok, err8 = [] * {} +_ok, err9 = [] * r'foo' +_ok, err10 = [] * now() array_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = {} * "foo" -ok, err2 = {} * 1 -ok, err3 = {} * 1.0 -ok, err4 = {} * true -ok, err5 = {} * false -ok, err6 = {} * null -ok, err7 = {} * [] -ok, err8 = {} * {} -ok, err9 = {} * r'foo' -ok, err10 = {} * now() +_ok, err1 = {} * "foo" +_ok, err2 = {} * 1 +_ok, err3 = {} * 1.0 +_ok, err4 = {} * true +_ok, err5 = {} * false +_ok, err6 = {} * null +_ok, err7 = {} * [] +_ok, err8 = {} * {} +_ok, err9 = {} * r'foo' +_ok, err10 = {} * now() object_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = r'' * "foo" -ok, err2 = r'' * 1 -ok, err3 = r'' * 1.0 -ok, err4 = r'' * true -ok, err5 = r'' * false -ok, err6 = r'' * null -ok, err7 = r'' * [] -ok, err8 = r'' * {} -ok, err9 = r'' * r'foo' -ok, err10 = r'' * now() +_ok, err1 = r'' * "foo" +_ok, err2 = r'' * 1 +_ok, err3 = r'' * 1.0 +_ok, err4 = r'' * true +_ok, err5 = r'' * false +_ok, err6 = r'' * null +_ok, err7 = r'' * [] +_ok, err8 = r'' * {} +_ok, err9 = r'' * r'foo' +_ok, err10 = r'' * now() regex_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = from_unix_timestamp(0) * "foo" -ok, err2 = from_unix_timestamp(0) * 1 -ok, err3 = from_unix_timestamp(0) * 1.0 -ok, err4 = from_unix_timestamp(0) * true -ok, err5 = from_unix_timestamp(0) * false -ok, err6 = from_unix_timestamp(0) * null -ok, err7 = from_unix_timestamp(0) * [] -ok, err8 = from_unix_timestamp(0) * {} -ok, err9 = from_unix_timestamp(0) * r'foo' -ok, err10 = from_unix_timestamp(0) * now() +_ok, err1 = from_unix_timestamp(0) * "foo" +_ok, err2 = from_unix_timestamp(0) * 1 +_ok, err3 = from_unix_timestamp(0) * 1.0 +_ok, err4 = from_unix_timestamp(0) * true +_ok, err5 = from_unix_timestamp(0) * false +_ok, err6 = from_unix_timestamp(0) * null +_ok, err7 = from_unix_timestamp(0) * [] +_ok, err8 = from_unix_timestamp(0) * {} +_ok, err9 = from_unix_timestamp(0) * r'foo' +_ok, err10 = from_unix_timestamp(0) * now() timestamp_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] diff --git a/lib/tests/tests/expressions/arithmetic/subtraction/invalid.vrl b/lib/tests/tests/expressions/arithmetic/subtraction/invalid.vrl index 1253dd1885..1e755b6415 100644 --- a/lib/tests/tests/expressions/arithmetic/subtraction/invalid.vrl +++ b/lib/tests/tests/expressions/arithmetic/subtraction/invalid.vrl @@ -95,116 +95,116 @@ # "can't subtract type timestamp from timestamp" # ] -ok, err1 = 1 - "foo" -ok, err2 = 1 - true -ok, err3 = 1 - false -ok, err4 = 1 - null -ok, err5 = 1 - [] -ok, err6 = 1 - {} -ok, err7 = 1 - r'foo' -ok, err8 = 1 - now() +_ok, err1 = 1 - "foo" +_ok, err2 = 1 - true +_ok, err3 = 1 - false +_ok, err4 = 1 - null +_ok, err5 = 1 - [] +_ok, err6 = 1 - {} +_ok, err7 = 1 - r'foo' +_ok, err8 = 1 - now() integer_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = 1.0 - "foo" -ok, err2 = 1.0 - true -ok, err3 = 1.0 - false -ok, err4 = 1.0 - null -ok, err5 = 1.0 - [] -ok, err6 = 1.0 - {} -ok, err7 = 1.0 - r'foo' -ok, err8 = 1.0 - now() +_ok, err1 = 1.0 - "foo" +_ok, err2 = 1.0 - true +_ok, err3 = 1.0 - false +_ok, err4 = 1.0 - null +_ok, err5 = 1.0 - [] +_ok, err6 = 1.0 - {} +_ok, err7 = 1.0 - r'foo' +_ok, err8 = 1.0 - now() float_errs = [err1, err2, err3, err4, err5, err6, err7, err8] -ok, err1 = "foo" - "foo" -ok, err2 = "foo" - 1 -ok, err3 = "foo" - 1.0 -ok, err4 = "foo" - true -ok, err5 = "foo" - false -ok, err6 = "foo" - null -ok, err7 = "foo" - [] -ok, err8 = "foo" - {} -ok, err9 = "foo" - r'foo' -ok, err10 = "foo" - now() +_ok, err1 = "foo" - "foo" +_ok, err2 = "foo" - 1 +_ok, err3 = "foo" - 1.0 +_ok, err4 = "foo" - true +_ok, err5 = "foo" - false +_ok, err6 = "foo" - null +_ok, err7 = "foo" - [] +_ok, err8 = "foo" - {} +_ok, err9 = "foo" - r'foo' +_ok, err10 = "foo" - now() string_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = true - "foo" -ok, err2 = true - 1 -ok, err3 = true - 1.0 -ok, err4 = true - true -ok, err5 = true - false -ok, err6 = true - null -ok, err7 = true - [] -ok, err8 = true - {} -ok, err9 = true - r'foo' -ok, err10 = true - now() +_ok, err1 = true - "foo" +_ok, err2 = true - 1 +_ok, err3 = true - 1.0 +_ok, err4 = true - true +_ok, err5 = true - false +_ok, err6 = true - null +_ok, err7 = true - [] +_ok, err8 = true - {} +_ok, err9 = true - r'foo' +_ok, err10 = true - now() bool_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = null - "foo" -ok, err2 = null - 1 -ok, err3 = null - 1.0 -ok, err4 = null - true -ok, err5 = null - false -ok, err6 = null - null -ok, err7 = null - [] -ok, err8 = null - {} -ok, err9 = null - r'foo' -ok, err10 = null - now() +_ok, err1 = null - "foo" +_ok, err2 = null - 1 +_ok, err3 = null - 1.0 +_ok, err4 = null - true +_ok, err5 = null - false +_ok, err6 = null - null +_ok, err7 = null - [] +_ok, err8 = null - {} +_ok, err9 = null - r'foo' +_ok, err10 = null - now() null_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = [] - "foo" -ok, err2 = [] - 1 -ok, err3 = [] - 1.0 -ok, err4 = [] - true -ok, err5 = [] - false -ok, err6 = [] - null -ok, err7 = [] - [] -ok, err8 = [] - {} -ok, err9 = [] - r'foo' -ok, err10 = [] - now() +_ok, err1 = [] - "foo" +_ok, err2 = [] - 1 +_ok, err3 = [] - 1.0 +_ok, err4 = [] - true +_ok, err5 = [] - false +_ok, err6 = [] - null +_ok, err7 = [] - [] +_ok, err8 = [] - {} +_ok, err9 = [] - r'foo' +_ok, err10 = [] - now() array_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = {} - "foo" -ok, err2 = {} - 1 -ok, err3 = {} - 1.0 -ok, err4 = {} - true -ok, err5 = {} - false -ok, err6 = {} - null -ok, err7 = {} - [] -ok, err8 = {} - {} -ok, err9 = {} - r'foo' -ok, err10 = {} - now() +_ok, err1 = {} - "foo" +_ok, err2 = {} - 1 +_ok, err3 = {} - 1.0 +_ok, err4 = {} - true +_ok, err5 = {} - false +_ok, err6 = {} - null +_ok, err7 = {} - [] +_ok, err8 = {} - {} +_ok, err9 = {} - r'foo' +_ok, err10 = {} - now() object_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = r'' - "foo" -ok, err2 = r'' - 1 -ok, err3 = r'' - 1.0 -ok, err4 = r'' - true -ok, err5 = r'' - false -ok, err6 = r'' - null -ok, err7 = r'' - [] -ok, err8 = r'' - {} -ok, err9 = r'' - r'foo' -ok, err10 = r'' - now() +_ok, err1 = r'' - "foo" +_ok, err2 = r'' - 1 +_ok, err3 = r'' - 1.0 +_ok, err4 = r'' - true +_ok, err5 = r'' - false +_ok, err6 = r'' - null +_ok, err7 = r'' - [] +_ok, err8 = r'' - {} +_ok, err9 = r'' - r'foo' +_ok, err10 = r'' - now() regex_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] -ok, err1 = from_unix_timestamp(0) - "foo" -ok, err2 = from_unix_timestamp(0) - 1 -ok, err3 = from_unix_timestamp(0) - 1.0 -ok, err4 = from_unix_timestamp(0) - true -ok, err5 = from_unix_timestamp(0) - false -ok, err6 = from_unix_timestamp(0) - null -ok, err7 = from_unix_timestamp(0) - [] -ok, err8 = from_unix_timestamp(0) - {} -ok, err9 = from_unix_timestamp(0) - r'foo' -ok, err10 = from_unix_timestamp(0) - now() +_ok, err1 = from_unix_timestamp(0) - "foo" +_ok, err2 = from_unix_timestamp(0) - 1 +_ok, err3 = from_unix_timestamp(0) - 1.0 +_ok, err4 = from_unix_timestamp(0) - true +_ok, err5 = from_unix_timestamp(0) - false +_ok, err6 = from_unix_timestamp(0) - null +_ok, err7 = from_unix_timestamp(0) - [] +_ok, err8 = from_unix_timestamp(0) - {} +_ok, err9 = from_unix_timestamp(0) - r'foo' +_ok, err10 = from_unix_timestamp(0) - now() timestamp_errs = [err1, err2, err3, err4, err5, err6, err7, err8, err9, err10] diff --git a/lib/tests/tests/expressions/assignment/infallible_assignment_typedef.vrl b/lib/tests/tests/expressions/assignment/infallible_assignment_typedef.vrl index 638eaf85e5..342a8cc035 100644 --- a/lib/tests/tests/expressions/assignment/infallible_assignment_typedef.vrl +++ b/lib/tests/tests/expressions/assignment/infallible_assignment_typedef.vrl @@ -1,3 +1,3 @@ # result: { "array": { "0": { "integer": true } }, "bytes": true } maybe_array = if true {null} else { [] } -type_def({x,err = push(maybe_array, 1)}) +type_def({_x, _err = push(maybe_array, 1)}) diff --git a/lib/tests/tests/expressions/assignment/infallible_err_return_value.vrl b/lib/tests/tests/expressions/assignment/infallible_err_return_value.vrl index 661aeea4c5..4de11cb73a 100644 --- a/lib/tests/tests/expressions/assignment/infallible_err_return_value.vrl +++ b/lib/tests/tests/expressions/assignment/infallible_err_return_value.vrl @@ -1,3 +1,3 @@ # result: "can't divide by zero" -ok, err = 1 / 0 +_ok, _err = 1 / 0 diff --git a/lib/tests/tests/expressions/assignment/infallible_ok_return_value.vrl b/lib/tests/tests/expressions/assignment/infallible_ok_return_value.vrl index 02186cabf9..dd6677c53e 100644 --- a/lib/tests/tests/expressions/assignment/infallible_ok_return_value.vrl +++ b/lib/tests/tests/expressions/assignment/infallible_ok_return_value.vrl @@ -1,4 +1,4 @@ # result: 1.0 .x = 1 -ok, err = 1 / .x +_ok, _err = 1 / .x diff --git a/lib/tests/tests/expressions/assignment/single_return_value.vrl b/lib/tests/tests/expressions/assignment/single_return_value.vrl index 5913bc4d6a..d965816612 100644 --- a/lib/tests/tests/expressions/assignment/single_return_value.vrl +++ b/lib/tests/tests/expressions/assignment/single_return_value.vrl @@ -1,3 +1,3 @@ # result: 5 -foo = 5 +_foo = 5 diff --git a/lib/tests/tests/expressions/block/abort_check.vrl b/lib/tests/tests/expressions/block/abort_check.vrl index f721bce27c..9e6d30ef1e 100644 --- a/lib/tests/tests/expressions/block/abort_check.vrl +++ b/lib/tests/tests/expressions/block/abort_check.vrl @@ -3,6 +3,6 @@ .a = "string" { - x = .a + _x = .a .a = abort } diff --git a/lib/tests/tests/expressions/block/fallibility_check.vrl b/lib/tests/tests/expressions/block/fallibility_check.vrl index 1fe61a2835..583322c080 100644 --- a/lib/tests/tests/expressions/block/fallibility_check.vrl +++ b/lib/tests/tests/expressions/block/fallibility_check.vrl @@ -2,7 +2,7 @@ # result: {"a": {}} .a = "string" { - x = to_string(.a) + _x = to_string(.a) .a = {} } . diff --git a/lib/tests/tests/expressions/block/multiple_expressions_multi_line.vrl b/lib/tests/tests/expressions/block/multiple_expressions_multi_line.vrl index 02ec3c63df..347bff1d01 100644 --- a/lib/tests/tests/expressions/block/multiple_expressions_multi_line.vrl +++ b/lib/tests/tests/expressions/block/multiple_expressions_multi_line.vrl @@ -2,6 +2,6 @@ { foo = 5 - 3 + _x = 3 1 + foo } diff --git a/lib/tests/tests/expressions/block/multiple_expressions_single_line.vrl b/lib/tests/tests/expressions/block/multiple_expressions_single_line.vrl index 5e521950f3..d1b2c976b0 100644 --- a/lib/tests/tests/expressions/block/multiple_expressions_single_line.vrl +++ b/lib/tests/tests/expressions/block/multiple_expressions_single_line.vrl @@ -1,3 +1,3 @@ # result: 6 -{ foo = 5; 3; 1 + foo } +{ foo = 5; foo - 5; 1 + foo } diff --git a/lib/tests/tests/expressions/function_call/closure_scope.vrl b/lib/tests/tests/expressions/function_call/closure_scope.vrl index 1bdc99b57a..c97ca1ca90 100644 --- a/lib/tests/tests/expressions/function_call/closure_scope.vrl +++ b/lib/tests/tests/expressions/function_call/closure_scope.vrl @@ -12,7 +12,7 @@ # = see language documentation at https://vrl.dev # = try your code in the VRL REPL, learn more at https://vrl.dev/examples -for_each({ "foo": "bar" }) -> |key, value| { +for_each({ "foo": "bar" }) -> |key, _value| { key = upcase(key) zoog = 72 } diff --git a/lib/tests/tests/expressions/function_call/closure_scope_inheritance.vrl b/lib/tests/tests/expressions/function_call/closure_scope_inheritance.vrl index 6b63d45887..9e02b58881 100644 --- a/lib/tests/tests/expressions/function_call/closure_scope_inheritance.vrl +++ b/lib/tests/tests/expressions/function_call/closure_scope_inheritance.vrl @@ -3,7 +3,7 @@ key = "yay" zoog = "yay" -for_each({ "foo": "bar" }) -> |key, value| { +for_each({ "foo": "bar" }) -> |key, _value| { key = "boo" zoog = "boo" } diff --git a/lib/tests/tests/expressions/if_statement/conditional_assignment.vrl b/lib/tests/tests/expressions/if_statement/conditional_assignment.vrl index 78e2baf755..a3ec951a82 100644 --- a/lib/tests/tests/expressions/if_statement/conditional_assignment.vrl +++ b/lib/tests/tests/expressions/if_statement/conditional_assignment.vrl @@ -6,7 +6,7 @@ # } . = {} -result = if true { +_result = if true { .x = true } else { .y = "hi" diff --git a/lib/tests/tests/expressions/literal/array.vrl b/lib/tests/tests/expressions/literal/array.vrl index 9a0a18c8d0..5eca5e65e6 100644 --- a/lib/tests/tests/expressions/literal/array.vrl +++ b/lib/tests/tests/expressions/literal/array.vrl @@ -1,20 +1,22 @@ # result: [true, 1, "bar"] # empty -[ ] +_arr = [ ] # single -[true] -[true,] +_arr = [true] +_arr = [true,] # multiple -[true, 1, "bar"] -[true, 1, "bar",] -[ +_arr = [true, 1, "bar"] +_arr = [true, 1, "bar",] + +_arr = [ true, 1, "bar", ] + [ true, 1, diff --git a/lib/tests/tests/expressions/literal/object.vrl b/lib/tests/tests/expressions/literal/object.vrl index b0ea6d864b..715f2cadfb 100644 --- a/lib/tests/tests/expressions/literal/object.vrl +++ b/lib/tests/tests/expressions/literal/object.vrl @@ -1,19 +1,20 @@ # result: { "bar": 12.0, "foo": null } # empty -{ } +_o = { } # single -{ "foo": null } -{ "foo": null, } +_o = { "foo": null } +_o = { "foo": null, } # multiple -{ "foo": null, "bar": 12.0 } -{ "foo": null, "bar": 12.0, } -{ +_o = { "foo": null, "bar": 12.0 } +_o = { "foo": null, "bar": 12.0, } +_o = { "foo": null, "bar": 12.0, } + { "foo": null, "bar": 12.0 diff --git a/lib/tests/tests/expressions/short_circuit/and.vrl b/lib/tests/tests/expressions/short_circuit/and.vrl index 7e427f33f5..5d9a918440 100644 --- a/lib/tests/tests/expressions/short_circuit/and.vrl +++ b/lib/tests/tests/expressions/short_circuit/and.vrl @@ -5,10 +5,11 @@ # } # This should be infallible (even though the right-hand side is fallible) -false && (1/0) +_x = false && (1/0) x = false -false && {x = "string"} +_x = false && {x = "string"} + # The typedef of the RHS is not applied (since it is guaranteed to short-circuit) result.a = type_def(x) assert!(type_def(x) == { "boolean": true }) @@ -19,7 +20,7 @@ result.b = type_def(x) assert!(type_def(x) == { "boolean": true, "bytes": true }) # The typedef of RHS is always applied -false || {x = "string"; true} +_x = false || {x = "string"; true} result.c = type_def(x) assert!(type_def(x) == {"bytes": true }) diff --git a/lib/tests/tests/expressions/short_circuit/or.vrl b/lib/tests/tests/expressions/short_circuit/or.vrl index 4ff41d7bcb..19f786f25d 100644 --- a/lib/tests/tests/expressions/short_circuit/or.vrl +++ b/lib/tests/tests/expressions/short_circuit/or.vrl @@ -1,11 +1,11 @@ # result: true # This should be infallible (even though the right-hand side is fallible) -true || (1/0) +_x = true || (1/0) x = false -true || {x = "string"} +_x = true || {x = "string"} # The typedef of the RHS is not applied (since it is guaranteed to short-circuit) assert!(type_def(x) == { "boolean": true }) @@ -16,5 +16,5 @@ assert!(type_def(x) == { "boolean": true, "bytes": true }) # The typedef of RHS is always applied -true && {x = "string"; true} +_x = true && {x = "string"; true} assert!(type_def(x) == {"bytes": true }) diff --git a/lib/tests/tests/functions/parse_etld/etld_from_url.vrl b/lib/tests/tests/functions/parse_etld/etld_from_url.vrl new file mode 100644 index 0000000000..3a3850ba66 --- /dev/null +++ b/lib/tests/tests/functions/parse_etld/etld_from_url.vrl @@ -0,0 +1,6 @@ +# object: { "url": "https://vector.dev" } +# result: { "etld": "dev", "etld_plus": "vector.dev", "known_suffix": true } + +parsed_url = parse_url!(.url) +etld_result = parse_etld!(parsed_url.host, plus_parts: 1) +etld_result diff --git a/lib/tests/tests/functions/parse_etld/etld_plus_one.vrl b/lib/tests/tests/functions/parse_etld/etld_plus_one.vrl new file mode 100644 index 0000000000..cccef5cdf7 --- /dev/null +++ b/lib/tests/tests/functions/parse_etld/etld_plus_one.vrl @@ -0,0 +1,5 @@ +# object: { "host": "vector.dev" } +# result: { "etld": "dev", "etld_plus": "vector.dev", "known_suffix": true } + +etld_result = parse_etld!(.host, plus_parts: 1) +etld_result diff --git a/lib/tests/tests/functions/parse_etld/etld_plus_ten.vrl b/lib/tests/tests/functions/parse_etld/etld_plus_ten.vrl new file mode 100644 index 0000000000..b4d42cab41 --- /dev/null +++ b/lib/tests/tests/functions/parse_etld/etld_plus_ten.vrl @@ -0,0 +1,5 @@ +# object: { "host": "vector.dev" } +# result: { "etld": "dev", "etld_plus": "vector.dev", "known_suffix": true } + +etld_result = parse_etld!(.host, plus_parts: 10) +etld_result diff --git a/lib/tests/tests/functions/parse_etld/just_etld.vrl b/lib/tests/tests/functions/parse_etld/just_etld.vrl new file mode 100644 index 0000000000..8c8086d382 --- /dev/null +++ b/lib/tests/tests/functions/parse_etld/just_etld.vrl @@ -0,0 +1,6 @@ +# object: { "host": "vector.dev" } +# result: { "etld": "dev", "known_suffix": true } + +etld_result = parse_etld!(.host) +del(etld_result.etld_plus) +etld_result diff --git a/lib/tests/tests/functions/parse_groks.vrl b/lib/tests/tests/functions/parse_groks.vrl new file mode 100644 index 0000000000..f6f130b402 --- /dev/null +++ b/lib/tests/tests/functions/parse_groks.vrl @@ -0,0 +1,8 @@ +# object: { "message": "username=Hupsakee" } +# result: { "username": "Hupsakee" } + +. = parse_groks!( + .message, + patterns: [ "%{TEST}" ], + alias_sources: [ "lib/tests/tests/functions/parse_groks_alias_source.json" ] +) diff --git a/lib/tests/tests/functions/parse_groks_alias_source.json b/lib/tests/tests/functions/parse_groks_alias_source.json new file mode 100644 index 0000000000..bd273dda5a --- /dev/null +++ b/lib/tests/tests/functions/parse_groks_alias_source.json @@ -0,0 +1,4 @@ +{ + "TEST": "%{OTHER_TEST}", + "OTHER_TEST": "username=%{USERNAME:username}" +} diff --git a/lib/tests/tests/functions/punycode/decode.vrl b/lib/tests/tests/functions/punycode/decode.vrl new file mode 100644 index 0000000000..1e609546b3 --- /dev/null +++ b/lib/tests/tests/functions/punycode/decode.vrl @@ -0,0 +1,3 @@ +# result: "www.porquénopuedensimplementehablarenespañol.com" + +decode_punycode!("www.xn--PorqunopuedensimplementehablarenEspaol-fmd56a.com") diff --git a/lib/tests/tests/functions/punycode/encode.vrl b/lib/tests/tests/functions/punycode/encode.vrl new file mode 100644 index 0000000000..9db41b5297 --- /dev/null +++ b/lib/tests/tests/functions/punycode/encode.vrl @@ -0,0 +1,3 @@ +# result: "www.xn--ihqwcrb4cv8a8dqg056pqjye.com" + +encode_punycode!("www.他们为什么不说中文.com") diff --git a/lib/tests/tests/functions/punycode/encode_decode.vrl b/lib/tests/tests/functions/punycode/encode_decode.vrl new file mode 100644 index 0000000000..cb02279452 --- /dev/null +++ b/lib/tests/tests/functions/punycode/encode_decode.vrl @@ -0,0 +1,5 @@ +# object: { "host": "www.ليهمابتكلموشعربي؟.他们为什么不说中文" } +# result: "www.ليهمابتكلموشعربي؟.他们为什么不说中文" + +encoded = encode_punycode!(.host) +decode_punycode!(encoded) diff --git a/lib/tests/tests/functions/punycode/url_encode_decode.vrl b/lib/tests/tests/functions/punycode/url_encode_decode.vrl new file mode 100644 index 0000000000..936a972f1e --- /dev/null +++ b/lib/tests/tests/functions/punycode/url_encode_decode.vrl @@ -0,0 +1,12 @@ +# object: { "url": "https://www.CAFé.com" } +# result: { "host": "www.xn--caf-dma.com", "host_decoded": "www.café.com" } + +# parse url +parsed_url = parse_url!(.url) + +# delete url - no longer needed +del(.url) + +.host = parsed_url.host +.host_decoded = decode_punycode!(.host) +. diff --git a/lib/tests/tests/functions/replace_with/capture_groups.vrl b/lib/tests/tests/functions/replace_with/capture_groups.vrl new file mode 100644 index 0000000000..8bf903a65d --- /dev/null +++ b/lib/tests/tests/functions/replace_with/capture_groups.vrl @@ -0,0 +1,3 @@ +# result: F bar F F fo F G + +replace_with("foo bar faa fee fo fum gum", r'([fg])\w\w') -> |m| { upcase(string!(m.captures[0])) } diff --git a/lib/tests/tests/functions/replace_with/captures_capture_group_name.vrl b/lib/tests/tests/functions/replace_with/captures_capture_group_name.vrl new file mode 100644 index 0000000000..415e1b9abb --- /dev/null +++ b/lib/tests/tests/functions/replace_with/captures_capture_group_name.vrl @@ -0,0 +1,4 @@ +# result: +# function call error for "replace_with" at (1:73): Capture group cannot be named "string" or "captures" + +replace_with("captain bold", r'cap(?P\w*)') -> |_m| { "test" } diff --git a/lib/tests/tests/functions/replace_with/error_in_closure.vrl b/lib/tests/tests/functions/replace_with/error_in_closure.vrl new file mode 100644 index 0000000000..668384b472 --- /dev/null +++ b/lib/tests/tests/functions/replace_with/error_in_closure.vrl @@ -0,0 +1,7 @@ +# result: +# function call error for "replace_with" at (1:105): function call error for "assert" at (59:92): failed to parse + +replace_with("this is a test", r'(?i)test') -> |_m| { + assert!(false, "failed to parse") + "TEST" +} diff --git a/lib/tests/tests/functions/replace_with/neg_count.vrl b/lib/tests/tests/functions/replace_with/neg_count.vrl new file mode 100644 index 0000000000..42d2b717c5 --- /dev/null +++ b/lib/tests/tests/functions/replace_with/neg_count.vrl @@ -0,0 +1,3 @@ +# result: "fOO bAr cAt dOg" + +replace_with("foo bar cat dog", r'[oa]*', count: -32) -> |m| { upcase(m.string) } diff --git a/lib/tests/tests/functions/replace_with/string_capture_group_name.vrl b/lib/tests/tests/functions/replace_with/string_capture_group_name.vrl new file mode 100644 index 0000000000..954d79d61d --- /dev/null +++ b/lib/tests/tests/functions/replace_with/string_capture_group_name.vrl @@ -0,0 +1,4 @@ +# result: +# function call error for "replace_with" at (1:64): Capture group cannot be named "string" or "captures" + +replace_with("a test", r'"(?P.*)"') -> |m| { m.string } diff --git a/lib/tests/tests/functions/replace_with/wrong_type.vrl b/lib/tests/tests/functions/replace_with/wrong_type.vrl new file mode 100644 index 0000000000..56eab0c810 --- /dev/null +++ b/lib/tests/tests/functions/replace_with/wrong_type.vrl @@ -0,0 +1,15 @@ +# result: +# error[E122]: type mismatch in closure return type +# ┌─ :2:34 +# │ +# 2 │ replace_with("", r'test') -> |m| { to_int!(m.string) } +# │ ^^^^^^^^^^^^^^^^^^^^^ +# │ │ +# │ block returns invalid value type +# │ received: integer +# │ expected: string +# │ +# = see language documentation at https://vrl.dev +# = try your code in the VRL REPL, learn more at https://vrl.dev/examples + +replace_with("", r'test') -> |m| { to_int!(m.string) } diff --git a/lib/tests/tests/functions/replace_with/zero_count.vrl b/lib/tests/tests/functions/replace_with/zero_count.vrl new file mode 100644 index 0000000000..9569476c06 --- /dev/null +++ b/lib/tests/tests/functions/replace_with/zero_count.vrl @@ -0,0 +1,3 @@ +#result: "foo bar" + +replace_with("foo bar", r'[oa]', count: 0) -> |m| { upcase(m.string) } diff --git a/lib/tests/tests/functions/slice_typedef.vrl b/lib/tests/tests/functions/slice_typedef.vrl index ce3f6bfabe..8a8a81b1a1 100644 --- a/lib/tests/tests/functions/slice_typedef.vrl +++ b/lib/tests/tests/functions/slice_typedef.vrl @@ -4,5 +4,5 @@ a = [] a[0] = {"foo": {"a": 1}} a[1] = {"foo": {"a": "bytes"}} -{ "foo": { "a": "bytes" } } +_x = { "foo": { "a": "bytes" } } type_def(slice!(a, 0)) diff --git a/lib/tests/tests/internal/allow_reserved_keywords.vrl b/lib/tests/tests/internal/allow_reserved_keywords.vrl index 6d1e53aa0b..edf663e60f 100644 --- a/lib/tests/tests/internal/allow_reserved_keywords.vrl +++ b/lib/tests/tests/internal/allow_reserved_keywords.vrl @@ -1,12 +1,12 @@ # result: true # function names using reserved keywords are allowed -string(.foo) ?? "" +_x = string(.foo) ?? "" # partially reserved keywords in variables are allowed -if_foo = true -foo_else = true -abort_foo_for = true +_if_foo = true +_foo_else = true +_abort_foo_for = true # reserved keywords in paths are allowed .true = true diff --git a/lib/tests/tests/internal/assignment_rhs.vrl b/lib/tests/tests/internal/assignment_rhs.vrl index 0a392ea322..df4b632003 100644 --- a/lib/tests/tests/internal/assignment_rhs.vrl +++ b/lib/tests/tests/internal/assignment_rhs.vrl @@ -2,7 +2,7 @@ # result: { "foo": true, "bar": true, "baz": true } # Assignments as the rhs expression should be allowed -false || (.foo = true) -true && (.bar = true) -5 / 0 ?? (.baz = true) +_x = false || (.foo = true) +_x = true && (.bar = true) +_x = 5 / 0 ?? (.baz = true) . diff --git a/lib/tests/tests/internal/infallible_ok_maybe_null.vrl b/lib/tests/tests/internal/infallible_ok_maybe_null.vrl index 52d232b74a..878c53dfa1 100644 --- a/lib/tests/tests/internal/infallible_ok_maybe_null.vrl +++ b/lib/tests/tests/internal/infallible_ok_maybe_null.vrl @@ -2,5 +2,5 @@ # result: "BAR" .foo # any -.foo, err = to_string(.foo) # string (empty or "bar") +.foo, _err = to_string(.foo) # string (empty or "bar") .foo = upcase(.foo) # string diff --git a/lib/tests/tests/internal/query_ignore_parens_in_quotes.vrl b/lib/tests/tests/internal/query_ignore_parens_in_quotes.vrl index b019e020c1..992d43dd45 100644 --- a/lib/tests/tests/internal/query_ignore_parens_in_quotes.vrl +++ b/lib/tests/tests/internal/query_ignore_parens_in_quotes.vrl @@ -1,9 +1,9 @@ # result: null -r'\).' -upcase(").") -upcase(s').') -replace("", r'\).', "") +_x = r'\).' +_x = upcase(").") +_x = upcase(s').') +_x = replace("", r'\).', "") .foo.(bar | "baz)") .foo.(bar | "{foo}") .foo.(bar | "[{bar}]") diff --git a/lib/tests/tests/internal/short_circuit.vrl b/lib/tests/tests/internal/short_circuit.vrl index 542ae0b319..56c518cf9a 100644 --- a/lib/tests/tests/internal/short_circuit.vrl +++ b/lib/tests/tests/internal/short_circuit.vrl @@ -2,9 +2,11 @@ # result: { "foo": false, "bar": false, "baz": false } # The rhs condition should never trigger in these cases. -true || (.foo = true) -false && (.bar = true) .x = 1 -5 / .x ?? (.baz = true) +_arr = [ + true || (.foo = true), + false && (.bar = true), + 5 / .x ?? (.baz = true), +] del(.x) . diff --git a/lib/tests/tests/internal/type_def_merging.vrl b/lib/tests/tests/internal/type_def_merging.vrl index 56270945b1..c948247bcf 100644 --- a/lib/tests/tests/internal/type_def_merging.vrl +++ b/lib/tests/tests/internal/type_def_merging.vrl @@ -5,18 +5,18 @@ v1 = "foo" v1 = 1 -ceil(v1) +_x = ceil(v1) v2 = ["true"] v2 = [0] -ceil(v2[0]) +_x = ceil(v2[0]) v3 = ["true"] v3[0] = 0 -ceil(v3[0]) +_x = ceil(v3[0]) v4 = ["true"] v4[1] = 0 -upcase(v4[0]) +_x = upcase(v4[0]) [v1, v2, v3, v4] diff --git a/lib/tests/tests/issues/11287_http_pipelines_blackhole.vrl b/lib/tests/tests/issues/11287_http_pipelines_blackhole.vrl index bc06965878..c01082d29e 100644 --- a/lib/tests/tests/issues/11287_http_pipelines_blackhole.vrl +++ b/lib/tests/tests/issues/11287_http_pipelines_blackhole.vrl @@ -1,13 +1,13 @@ # issue: https://github.com/vectordotdev/vector/pull/11287 # result: [["bar"], ["qux"], { "bar": true }] -.a1, err = push(.a1, "foo") +.a1, _err = push(.a1, "foo") .a1 = push(.a1, "bar") -.a2, err = append(.a2, ["baz"]) +.a2, _err = append(.a2, ["baz"]) .a2 = append(.a2, ["qux"]) -.a3, err = merge(.a3, { "foo": true }) +.a3, _err = merge(.a3, { "foo": true }) .a3 = merge(.a3, { "bar": true }) [.a1, .a2, .a3] diff --git a/scripts/check_changelog_fragments.sh b/scripts/check_changelog_fragments.sh new file mode 100755 index 0000000000..fc7c99d765 --- /dev/null +++ b/scripts/check_changelog_fragments.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# This script is intended to run during CI, however it can be run locally by +# committing changelog fragments before executing the script. If the script +# finds an issue with your changelog fragment, you can un-stage the fragment +# from being committed and fix the issue. + +CHANGELOG_DIR="changelog.d" + +# NOTE: If these are altered, update both the 'changelog.d/README.md' and +# 'scripts/generate_release_chanbgelog.sh' accordingly. +FRAGMENT_TYPES="breaking|security|deprecation|feature|enhancement|fix" + +if [ ! -d "${CHANGELOG_DIR}" ]; then + echo "No ./${CHANGELOG_DIR} found. This tool must be invoked from the root of the repo." + exit 1 +fi + +# diff-filter=A lists only added files +FRAGMENTS=$(git diff --name-only --diff-filter=A origin/main ${CHANGELOG_DIR}) + +if [ -z "$FRAGMENTS" ]; then + echo "No changelog fragments detected" + echo "If no changes necessitate user-facing explanations, add the GH label 'no-changelog'" + echo "Otherwise, add changelog fragments to changelog.d/" + echo "For details, see 'changelog.d/README.md'" + exit 1 +fi + +# extract the basename from the file path +FRAGMENTS=$(xargs -n1 basename <<< "${FRAGMENTS}") + +# validate the fragments +while IFS= read -r fname; do + + if [[ ${fname} == "README.md" ]]; then + continue + fi + + echo "validating '${fname}'" + + IFS="." read -r -a arr <<< "${fname}" + + if [ "${#arr[@]}" -ne 3 ]; then + echo "invalid fragment filename: wrong number of period delimiters. expected '..md'. (${fname})" + exit 1 + fi + + if ! [[ "${arr[0]}" =~ ^[0-9]+$ ]]; then + echo "invalid fragment filename: first segment must be PR number. (${fname})" + exit 1 + fi + + if ! [[ "${arr[1]}" =~ ^(${FRAGMENT_TYPES})$ ]]; then + echo "invalid fragment filename: fragment type must be one of: (${FRAGMENT_TYPES}). (${fname})" + exit 1 + fi + + if [[ "${arr[2]}" != "md" ]]; then + echo "invalid fragment filename: extension must be markdown (.md): (${fname})" + exit 1 + fi + +done <<< "$FRAGMENTS" + +echo "changelog additions are valid." diff --git a/scripts/generate_release_changelog.sh b/scripts/generate_release_changelog.sh new file mode 100755 index 0000000000..3856d82e6a --- /dev/null +++ b/scripts/generate_release_changelog.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +set -o errexit + +README="README.md" +CHANGELOG="CHANGELOG.md" +CHANGELOG_DIR="changelog.d" +CHANGELOG_CFG="changelog.toml" + +ask_continue() { + while true; do + local choice + read -r choice + case $choice in + y) break; ;; + n) exit 1; ;; + *) echo "Please enter y or n"; ;; + esac + done +} + +cd $(dirname "$0")/.. +VRL_ROOT=$(pwd) + +VRL_VERSION=$(awk '/^version = "[0-9]+.[0-9]+.[0-9]+"/{print $3}' "${VRL_ROOT}"/Cargo.toml | tr -d '"') + +if ! [[ ${VRL_VERSION} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] ; then + echo "Error reading release version from Cargo.toml!" + exit 1 +else + echo + echo "[detected VRL release version: ${VRL_VERSION}]" +fi + +if ! python3 -m pip show towncrier > /dev/null ; then + echo "towncrier installation missing. Please install with 'python3 -m pip install towncrier'" + exit 1 +fi + +LOCAL_CHANGELOG_DIR=${VRL_ROOT}/${CHANGELOG_DIR} + +if [ ! -d "${LOCAL_CHANGELOG_DIR}" ]; then + echo "No ${CHANGELOG_DIR} found in ${VRL_ROOT}!" + exit 1 +fi + +LOCAL_CHANGELOG_CFG=${VRL_ROOT}/${CHANGELOG_CFG} + +if [ ! -f "${LOCAL_CHANGELOG_CFG}" ]; then + echo "No ${CHANGELOG_CFG} found in ${VRL_ROOT}!" + exit 1 +fi + +################################################################################ +echo +echo -n "[checking for changelog fragments..." + +HAVE_FRAGMENTS=false + +# changelog fragments that haven't been released are added at the root of the changelog dir. +for f in "${LOCAL_CHANGELOG_DIR}"/*.md ; do + if [[ $(basename "$f") == "${README}" ]] ; then + continue + fi + HAVE_FRAGMENTS=true + echo " done.]" + break +done + +if [ "${HAVE_FRAGMENTS}" = false ] ; then + echo " no changelog fragments were found! Exiting]" + exit 1 +fi + +################################################################################ + +echo +echo "[generating the changelog..." +if python3 -m towncrier build \ + --config "${LOCAL_CHANGELOG_CFG}" \ + --dir "${VRL_ROOT}" \ + --version "${VRL_VERSION}" \ + --keep +then + echo + echo " done]" +else + echo + echo " failed!]" + exit 1 +fi + +################################################################################ +echo +echo "[please review the ${CHANGELOG} for accuracy.]" + +echo +echo "[continue? The next step will retire changelog fragments being released by removing them from the repo (y/n)?]" + +ask_continue + +################################################################################ +echo +echo -n "[removing changelog fragments included in this release..." + +for f in "${LOCAL_CHANGELOG_DIR}"/*.md ; do + if [[ $(basename "$f") == "${README}" ]] ; then + continue + fi + if ! git rm -f "$f" ; then + echo "... failed to remove $f !]" + exit 1 + fi +done + +echo " done]" + +################################################################################ +echo +echo "[please review the changes to local VRL repo checkout and create a PR.]" diff --git a/scripts/publish.py b/scripts/publish.py old mode 100644 new mode 100755 index 25c65c05d3..f7e9524a0d --- a/scripts/publish.py +++ b/scripts/publish.py @@ -8,7 +8,7 @@ SCRIPTS_DIR = os.path.dirname(abspath(getsourcefile(lambda: 0))) REPO_ROOT_DIR = os.path.dirname(SCRIPTS_DIR) - +CHANGELOG_DIR = os.path.join(REPO_ROOT_DIR, "changelog.d") def get_crate_versions(crate_name): response = requests.get(f"https://crates.io/api/v1/crates/{crate_name}") @@ -32,25 +32,37 @@ def publish_vrl(version): tag_name = f"v{version}" tag_message = f"Release {version}" subprocess.run(["git", "tag", "-a", tag_name, "-m", tag_message], check=True, cwd=REPO_ROOT_DIR) - subprocess.run(["git", "push", "origin", version], check=True, cwd=REPO_ROOT_DIR) + subprocess.run(["git", "push", "origin", tag_name], check=True, cwd=REPO_ROOT_DIR) print(f"Tagged version.") except subprocess.CalledProcessError as e: print(f"An error occurred while publishing the crate: {e}") -def main(): +def assert_no_changelog_fragments(): + entries = os.listdir(CHANGELOG_DIR) + error = f"{CHANGELOG_DIR} should only contain a README.md file. Did you run ./scripts/generate_release_changelog.sh?" + assert len(entries) == 1, error + assert entries[0] == "README.md", error + + +def assert_version_is_not_published(current_version): crate_name = "vrl" versions = get_crate_versions(crate_name) print(f"Available versions for {crate_name}: {versions}") + if current_version in versions: + print(f"The version {current_version} is already published. Please update the version and try again.") + exit(1) + + +def main(): + assert_no_changelog_fragments() + toml_path = os.path.join(REPO_ROOT_DIR, "Cargo.toml") current_version = read_version_from_cargo_toml(toml_path) print(f"Current version in Cargo.toml: {current_version}") - - if current_version in versions: - print(f"The version {current_version} is already published. Please update the version and try again.") - else: - publish_vrl(current_version) + assert_version_is_not_published(current_version) + publish_vrl(current_version) if __name__ == "__main__": diff --git a/src/compiler/codes.rs b/src/compiler/codes.rs new file mode 100644 index 0000000000..a635657be6 --- /dev/null +++ b/src/compiler/codes.rs @@ -0,0 +1,2 @@ +// TODO Gather all codes here. Ensure they are unique and publicly documented. +pub const WARNING_UNUSED_CODE: usize = 900; diff --git a/src/compiler/expression.rs b/src/compiler/expression.rs index d235808636..a645659354 100644 --- a/src/compiler/expression.rs +++ b/src/compiler/expression.rs @@ -39,7 +39,7 @@ mod noop; mod not; mod object; mod op; -mod unary; +pub(crate) mod unary; mod variable; pub(crate) mod assignment; diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs index ea1a09d60f..93528afc15 100644 --- a/src/compiler/mod.rs +++ b/src/compiler/mod.rs @@ -1,31 +1,31 @@ #![deny( - // warnings, - clippy::all, - clippy::pedantic, - unreachable_pub, - unused_allocation, - unused_extern_crates, - unused_assignments, - unused_comparisons +// warnings, +clippy::all, +clippy::pedantic, +unreachable_pub, +unused_allocation, +unused_extern_crates, +unused_assignments, +unused_comparisons )] #![allow( - clippy::cast_possible_truncation, // allowed in initial deny commit - clippy::cast_possible_wrap, // allowed in initial deny commit - clippy::cast_precision_loss, // allowed in initial deny commit - clippy::cast_sign_loss, // allowed in initial deny commit - clippy::if_not_else, // allowed in initial deny commit - clippy::match_bool, // allowed in initial deny commit - clippy::match_same_arms, // allowed in initial deny commit - clippy::match_wild_err_arm, // allowed in initial deny commit - clippy::missing_errors_doc, // allowed in initial deny commit - clippy::missing_panics_doc, // allowed in initial deny commit - clippy::module_name_repetitions, // allowed in initial deny commit - clippy::needless_pass_by_value, // allowed in initial deny commit - clippy::return_self_not_must_use, // allowed in initial deny commit - clippy::semicolon_if_nothing_returned, // allowed in initial deny commit - clippy::similar_names, // allowed in initial deny commit - clippy::too_many_lines, // allowed in initial deny commit - let_underscore_drop, // allowed in initial deny commit +clippy::cast_possible_truncation, // allowed in initial deny commit +clippy::cast_possible_wrap, // allowed in initial deny commit +clippy::cast_precision_loss, // allowed in initial deny commit +clippy::cast_sign_loss, // allowed in initial deny commit +clippy::if_not_else, // allowed in initial deny commit +clippy::match_bool, // allowed in initial deny commit +clippy::match_same_arms, // allowed in initial deny commit +clippy::match_wild_err_arm, // allowed in initial deny commit +clippy::missing_errors_doc, // allowed in initial deny commit +clippy::missing_panics_doc, // allowed in initial deny commit +clippy::module_name_repetitions, // allowed in initial deny commit +clippy::needless_pass_by_value, // allowed in initial deny commit +clippy::return_self_not_must_use, // allowed in initial deny commit +clippy::semicolon_if_nothing_returned, // allowed in initial deny commit +clippy::similar_names, // allowed in initial deny commit +clippy::too_many_lines, // allowed in initial deny commit +let_underscore_drop, // allowed in initial deny commit )] use std::fmt::Debug; @@ -34,6 +34,7 @@ use std::{fmt::Display, str::FromStr}; pub use paste::paste; use serde::{Deserialize, Serialize}; +use crate::compiler::unused_expression_checker::check_for_unused_results; pub use compiler::{CompilationResult, Compiler}; pub use context::Context; pub use datetime::TimeZone; @@ -64,6 +65,7 @@ mod program; mod target; mod test_util; +pub mod codes; pub mod conversion; pub mod expression; pub mod function; @@ -71,6 +73,7 @@ pub mod prelude; pub mod runtime; pub mod state; pub mod type_def; +pub mod unused_expression_checker; pub mod value; pub type Result = std::result::Result; @@ -106,7 +109,16 @@ pub fn compile_with_state( let ast = parse(source) .map_err(|err| crate::diagnostic::DiagnosticList::from(vec![Box::new(err) as Box<_>]))?; - Compiler::compile(fns, ast, state, config) + let result = Compiler::compile(fns, ast.clone(), state, config); + let unused_warnings = check_for_unused_results(&ast); + if unused_warnings.is_empty() { + result + } else { + result.map(|mut compilation_result| { + compilation_result.warnings.extend(unused_warnings); + compilation_result + }) + } } /// Available VRL runtimes. diff --git a/src/compiler/unused_expression_checker.rs b/src/compiler/unused_expression_checker.rs new file mode 100644 index 0000000000..f395aa73e3 --- /dev/null +++ b/src/compiler/unused_expression_checker.rs @@ -0,0 +1,685 @@ +/// # Unused Expression Checker +/// +/// This module provides functionality for traversing VRL (Vector Remap Language) Abstract Syntax Trees (AST). +/// It's designed to detect and report unused expressions, helping users clean up and optimize their VRL scripts. +/// Initially, it will generate warnings for unused expressions. These warnings might be escalated to errors +/// in future versions, once the module has been battle-tested. +/// +/// ## How it works +/// +/// - **Traversal**: Recursively explores each node of the AST. This process begins after the program has +/// been successfully compiled. +/// - **Stateful Context**: Builds context on the fly to determine whether an expression is unused. The context +/// takes into account variable scopes, assignments, and the flow of the program. +/// - **Detection**: Identifies and reports expressions that do not contribute to assignments, +/// affect external events, or influence the outcome of function calls. +/// - **Ignored Variables**: Variable names prefixed with '_' are ignored. +/// +/// ## Caveats +/// - **Closures**: Closure support is minimal. For now, we are only ensuring that there are no false positives. +/// - **Variable Shadowing**: Variable shadowing is not supported. Unused variables will not be detected in this case. +use crate::compiler::codes::WARNING_UNUSED_CODE; +use crate::compiler::parser::{Ident, Node}; +use crate::diagnostic::{Diagnostic, DiagnosticList, Label, Note, Severity}; +use crate::parser::ast::{ + Array, Assignment, AssignmentOp, AssignmentTarget, Block, Container, Expr, FunctionCall, + IfStatement, Object, Predicate, QueryTarget, RootExpr, Unary, +}; +use crate::parser::template_string::StringSegment; +use crate::parser::{Literal, Program, Span}; +use std::collections::{BTreeMap, HashMap}; +use tracing::warn; + +#[must_use] +pub fn check_for_unused_results(ast: &Program) -> DiagnosticList { + let expression_visitor = AstVisitor { ast }; + expression_visitor.check_for_unused_results() +} + +pub struct AstVisitor<'a> { + ast: &'a Program, +} + +#[derive(Default, Debug, Clone)] +struct IdentState { + span: Span, + pending_usage: bool, +} + +#[derive(Default, Debug, Clone)] +struct VisitorState { + level: usize, + expecting_result: HashMap, + within_block_expression: HashMap, + ident_to_state: BTreeMap, + diagnostics: DiagnosticList, +} + +impl VisitorState { + fn is_unused(&self) -> bool { + let pending_result = self + .expecting_result + .get(&self.level) + .is_some_and(|active| *active); + !pending_result + } + + fn is_within_block(&self) -> bool { + self.within_block_expression + .get(&self.level) + .is_some_and(|within_block| *within_block) + } + + fn increase_level(&mut self) { + self.level += 1; + } + + fn decrease_level(&mut self) { + self.level -= 1; + } + + fn enter_block(&mut self) { + self.increase_level(); + self.within_block_expression.insert(self.level, true); + } + + fn exiting_block(&mut self) { + self.within_block_expression.insert(self.level, false); + self.decrease_level(); + } + + fn mark_level_as_expecting_result(&mut self) { + self.expecting_result.insert(self.level, true); + } + + fn mark_level_as_not_expecting_result(&mut self) { + self.expecting_result.insert(self.level, false); + } + + fn mark_identifier_pending_usage(&mut self, ident: &Ident, span: &Span) { + if ident.is_empty() || ident.starts_with('_') { + return; + } + + self.ident_to_state + .entry(ident.clone()) + .and_modify(|state| { + state.pending_usage = true; + }) + .or_insert(IdentState { + span: *span, + pending_usage: true, + }); + } + + fn mark_identifier_used(&mut self, ident: &Ident) { + if ident.is_empty() || ident.starts_with('_') { + return; + } + + if let Some(entry) = self.ident_to_state.get_mut(ident) { + entry.pending_usage = false; + } else { + warn!("unexpected identifier `{}` reported as used", ident); + } + } + + fn mark_query_target_pending_usage(&mut self, query_target: &Node) { + match &query_target.node { + QueryTarget::Internal(ident) => { + self.mark_identifier_pending_usage(ident, &query_target.span); + } + QueryTarget::External(_) => {} + QueryTarget::FunctionCall(_) => {} + QueryTarget::Container(_) => {} + } + } + + fn append_diagnostic(&mut self, message: String, span: &Span) { + self.diagnostics.push(Diagnostic { + severity: Severity::Warning, + code: WARNING_UNUSED_CODE, + message, + labels: vec![Label::primary( + "help: use the result of this expression or remove it", + span, + )], + notes: vec![Note::Basic( + "this expression has no side-effects".to_owned(), + )], + }) + } + + fn extend_diagnostics_for_unused_variables(&mut self) { + for (ident, state) in self.ident_to_state.clone() { + if state.pending_usage { + self.append_diagnostic(format!("unused variable `{ident}`"), &state.span); + } + } + } +} + +fn scoped_visit(state: &mut VisitorState, f: impl FnOnce(&mut VisitorState)) { + state.increase_level(); + state.mark_level_as_expecting_result(); + f(state); + state.mark_level_as_not_expecting_result(); + state.decrease_level(); +} + +impl AstVisitor<'_> { + fn visit_node(&self, node: &Node, state: &mut VisitorState) { + let expression = node.inner(); + + match expression { + Expr::Literal(literal) => { + if let Literal::String(template) = &literal.node { + for segment in &template.0 { + if let StringSegment::Template(ident, _) = segment { + state.mark_identifier_used(&Ident::from(ident.clone())); + } + } + } + if state.is_unused() { + state.append_diagnostic(format!("unused literal `{literal}`"), &node.span()); + } + } + Expr::Container(container) => { + self.visit_container(container, state); + } + Expr::IfStatement(if_statement) => { + scoped_visit(state, |state| { + self.visit_if_statement(if_statement, state); + }); + } + Expr::Op(op) => { + self.visit_node(&op.0, state); + scoped_visit(state, |state| { + self.visit_node(&op.2, state); + }); + } + Expr::Unary(unary) => match &unary.node { + Unary::Not(not) => { + self.visit_node(¬.1, state); + } + }, + Expr::Assignment(assignment) => { + self.visit_assignment(assignment, state); + } + Expr::Query(query) => match &query.node.target.node { + QueryTarget::Internal(ident) => { + if !state.is_unused() { + state.mark_identifier_used(ident); + } + } + QueryTarget::External(_) => {} + QueryTarget::FunctionCall(function_call) => { + self.visit_function_call(function_call, &query.node.target.span, state) + } + QueryTarget::Container(_) => {} + }, + Expr::FunctionCall(function_call) => { + self.visit_function_call(function_call, &function_call.span, state) + } + Expr::Variable(variable) => { + state.mark_identifier_used(&variable.node); + } + Expr::Abort(_) => {} + } + } + + fn visit_container(&self, node: &Node, state: &mut VisitorState) { + match &node.node { + Container::Group(group) => self.visit_node(&group.node.0, state), + Container::Block(block) => self.visit_block(block, state), + Container::Array(array) => self.visit_array(array, state), + Container::Object(object) => self.visit_object(object, state), + } + } + + fn visit_array(&self, array: &Node, state: &mut VisitorState) { + for expr in &array.0 { + self.visit_node(expr, state); + } + } + + fn visit_block(&self, block: &Node, state: &mut VisitorState) { + let block_expressions = &block.node.0; + if block_expressions.is_empty() { + return; + } + state.enter_block(); + + for (i, expr) in block_expressions.iter().enumerate() { + if i == block_expressions.len() - 1 { + state.exiting_block(); + } + self.visit_node(expr, state); + } + } + + fn visit_object(&self, object: &Node, state: &mut VisitorState) { + if state.is_unused() { + state.append_diagnostic(format!("unused object `{object}`"), &object.span); + } + for value in object.0.values() { + scoped_visit(state, |state| { + self.visit_node(value, state); + }); + } + } + + fn visit_if_statement(&self, if_statement: &Node, state: &mut VisitorState) { + match &if_statement.predicate.node { + Predicate::One(expr) => self.visit_node(expr, state), + Predicate::Many(exprs) => { + for expr in exprs { + self.visit_node(expr, state); + } + } + } + + scoped_visit(state, |state| { + self.visit_block(&if_statement.if_node, state); + }); + + if let Some(else_block) = &if_statement.else_node { + scoped_visit(state, |state| { + self.visit_block(else_block, state); + }); + } + } + + fn visit_assignment(&self, assignment: &Node, state: &mut VisitorState) { + state.increase_level(); + let level = state.level; + state.expecting_result.insert(level, true); + + // All targets needs to be used later. + let (op, targets) = match &assignment.node { + Assignment::Single { target, op, .. } => (op, vec![target]), + Assignment::Infallible { ok, err, op, .. } => (op, vec![ok, err]), + }; + for target in targets { + match &target.node { + AssignmentTarget::Noop => {} + AssignmentTarget::Query(query) => { + state.mark_query_target_pending_usage(&query.target); + } + AssignmentTarget::Internal(ident, path) => { + if *op == AssignmentOp::Assign && path.is_none() { + state.mark_identifier_pending_usage(ident, &target.span); + } else if *op == AssignmentOp::Merge { + // The following example: `x |= {}` falls under shadowing and is not handled. + state.mark_identifier_used(ident); + } + } + AssignmentTarget::External(_path) => {} + } + } + + // Visit the assignment right hand side. + match &assignment.node { + Assignment::Single { expr, .. } => { + self.visit_node(expr, state); + } + Assignment::Infallible { expr, .. } => { + self.visit_node(expr, state); + } + } + state.expecting_result.insert(level, false); + state.decrease_level(); + } + + fn visit_function_call( + &self, + function_call: &FunctionCall, + span: &Span, + state: &mut VisitorState, + ) { + for argument in &function_call.arguments { + state.increase_level(); + state.mark_level_as_expecting_result(); + self.visit_node(&argument.node.expr, state); + state.mark_level_as_not_expecting_result(); + state.decrease_level(); + } + + // This function call might be part of fallible block. + if !function_call.abort_on_error && state.is_within_block() { + state.mark_level_as_expecting_result(); + } + + match function_call.ident.0.as_str() { + // All bets are off for functions with side-effects. + "del" | "log" | "assert" | "assert_eq" => (), + _ => { + if let Some(closure) = &function_call.closure { + for variable in &closure.variables { + state.mark_identifier_pending_usage(&variable.node, &variable.span); + } + state.mark_level_as_expecting_result(); + self.visit_block(&closure.block, state); + state.mark_level_as_not_expecting_result(); + } else if state.is_unused() { + state.append_diagnostic( + format!("unused result for function call `{function_call}`"), + span, + ); + } + } + } + + if !function_call.abort_on_error && state.is_within_block() { + state.mark_level_as_not_expecting_result(); + } + } + + /// This function traverses the VRL AST and detects unused results. + /// An expression might have side-effects, in that case we do not except its result to be used. + /// + /// We want to detect the following cases: + /// * Unused Variables: a variable which is assigned a value but never used in any expression + /// * Unused Expressions: an expression without side-effects with an unused result + fn check_for_unused_results(&self) -> DiagnosticList { + let mut unused_warnings = DiagnosticList::default(); + let mut state = VisitorState::default(); + let root_expressions = &self.ast.0; + for (i, root_node) in root_expressions.iter().enumerate() { + let is_last = i == root_expressions.len() - 1; + if is_last { + state.increase_level(); + state.mark_level_as_expecting_result(); + } + match root_node.inner() { + RootExpr::Expr(node) => self.visit_node(node, &mut state), + RootExpr::Error(_) => {} + } + if is_last { + state.decrease_level(); + state.mark_level_as_not_expecting_result(); + } + } + state.extend_diagnostics_for_unused_variables(); + unused_warnings.extend(state.diagnostics); + unused_warnings + } +} + +#[cfg(test)] +mod test { + use crate::compiler::codes::WARNING_UNUSED_CODE; + use crate::stdlib; + use indoc::indoc; + + fn unused_test(source: &str, expected_warnings: Vec) { + let warnings = crate::compiler::compile(source, &stdlib::all()) + .unwrap() + .warnings; + + assert_eq!(warnings.len(), expected_warnings.len()); + + for (i, content) in expected_warnings.iter().enumerate() { + let warning = warnings.get(i).unwrap(); + assert_eq!(warning.code, WARNING_UNUSED_CODE); + assert!( + warning.message.contains(content), + "expected message `{}` to contain `{content}`", + warning.message + ); + } + } + + #[test] + fn unused_top_level_literal() { + let source = indoc! {r#" + "foo" + "program result" + "#}; + unused_test(source, vec![r#"unused literal `"foo"`"#.to_string()]); + } + + #[test] + fn unused_variable_in_assignment() { + let source = indoc! {r#" + foo = 5 + "#}; + unused_test(source, vec![r#"unused variable `foo`"#.to_string()]); + } + + #[test] + fn unused_literal() { + let source = indoc! {r#" + . = { + "unused" + "a" + } + "#}; + unused_test(source, vec![r#"unused literal `"unused"`"#.to_string()]); + } + + #[test] + fn unused_top_level_variable() { + let source = indoc! {r#" + x = "bar" + "#}; + unused_test(source, vec!["unused variable `x`".to_string()]); + } + + #[test] + fn test_nested_blocks() { + let source = indoc! {r#" + . = { + "1" + { + "2" + { + "3" + } + } + + . = {{{ x = 42; x }}} + + "4" + "5" + } + "#}; + + let expected_warnings: Vec = (1..5) + .map(|i| format!("unused literal `\"{i}\"`")) + .collect(); + unused_test(source, expected_warnings); + } + + #[test] + fn unused_object() { + let source = indoc! {r#" + .o = { "key": 1 } + { "array": [{"a": "b"}], "b": 2} + "program result" + "#}; + unused_test( + source, + vec![r#"unused object `{ "array": [{ "a": "b" }], "b": 2 }`"#.to_string()], + ); + } + + #[test] + fn unused_variables() { + let source = indoc! {r#" + a = "1" + b = { + c = "2" + "3" + } + d = random_bool() + . = d + "#}; + + let expected_warnings = ('a'..'d') + .map(|ident| format!("unused variable `{ident}`")) + .collect(); + unused_test(source, expected_warnings); + } + + #[test] + fn unused_function_result() { + let source = indoc! {r#" + .r = random_int(0,1) + random_bool() + "program result" + "#}; + unused_test( + source, + vec![r#"unused result for function call `random_bool()`"#.to_string()], + ); + } + + #[test] + fn unused_ident_with_path() { + let source = indoc! {r#" + x = {} + .f1 = x + y = {} + y.a = 1 + "#}; + unused_test(source, vec!["unused variable `y`".to_string()]); + } + + #[test] + fn unused_coalesce_result() { + let source = indoc! {r#" + parse_syslog("not syslog") ?? parse_common_log("not common") ?? "malformed" + .res = parse_syslog("not syslog") ?? parse_common_log("not common") ?? "malformed" + "#}; + unused_test( + source, + vec![r#"unused result for function call `parse_syslog("not syslog")`"#.to_string()], + ); + } + + #[test] + fn used_queries() { + let source = indoc! {r#" + _i_am_ignored = 42 + x = {} + x.foo = 1 + x.bar = 2 + .bar = remove!(x, ["foo"]).bar + + y = {"foo": 3}.foo + "#}; + unused_test(source, vec![r#"unused variable `y`"#.to_string()]); + } + + #[test] + fn used_in_if_condition() { + let source = indoc! {r#" + if starts_with!(.a, "foo") { + .a = "foo" + } else if starts_with!(.a, "bar") { + .a = "bar" + } + + x = 1 + .b = if (x < 1) { 0 } else { 1 } + + y = 2 + z = 3 + if (y < 2 && random_int(0, 4) < 3 ) { 0 } else { .c = z } + + x = {} + x.a = 1 + .d = if (x.a < 1) { 0 } else { 1 } + "#}; + unused_test(source, vec![]); + } + + #[test] + fn used_in_function_arguments() { + let source = indoc! {r#" + x = {} + x.foo = 1 + .r = random_int!({x.foo}, x.foo + 1) + + x.bar = 2 + exists(field: x.bar) + del(x.bar, compact: false) + "#}; + unused_test( + source, + vec![r#"unused result for function call `exists(field: xbar)`"#.to_string()], + ); + } + + #[test] + fn closure_shadows_unused_variable() { + let source = indoc! {r#" + count = 0; + value = 42 + for_each({ "a": 1, "b": 2 }) -> |_key, value| { count = count + value }; + count + "#}; + // Note that the `value` outside of the closure block is unused but not detected. + unused_test(source, vec![]); + } + + #[test] + fn used_closure_result() { + let source = indoc! {r#" + patterns = [r'foo', r'bar'] + matched = false + for_each(patterns) -> |_, pattern| { + if !matched && match!(.message, pattern) { + matched = true + } + } + matched + "#}; + // Note that the `value` outside of the closure block is unused but not detected. + unused_test(source, vec![]); + } + + #[test] + fn used_function_result_in_fallible_block() { + let source = indoc! {r#" + { + parse_json("invalid") + 2 + } ?? 1 + "#}; + unused_test(source, vec![]); + } + + #[test] + fn unused_shadow_variable_not_detected() { + // TODO: Support variable shadowing. A potential solution is to introduce the following type: + // type IdentState = HashMap; + let source = indoc! {r#" + x = 1 + x = 2 + { + x = { + x = { + x = 3 + 4 + } + x + } + x + } + "#}; + unused_test(source, vec![]); + } + + #[test] + fn undetected_merge_assignment() { + // `x` is not used after the merging operation. This case is not detected. + let source = indoc! {r#" + x = {} + x |= { "a" : 1} + . + "#}; + unused_test(source, vec![]); + } +} diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 0aa5820066..52cdbc3cbd 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -138,7 +138,7 @@ impl Hash for Node { // program // ----------------------------------------------------------------------------- -#[derive(PartialEq)] +#[derive(Clone, PartialEq)] pub struct Program(pub Vec>); impl fmt::Debug for Program { @@ -183,7 +183,7 @@ impl IntoIterator for Program { // ----------------------------------------------------------------------------- #[allow(clippy::large_enum_variant)] -#[derive(PartialEq)] +#[derive(Clone, PartialEq)] pub enum RootExpr { Expr(Node), @@ -285,7 +285,7 @@ impl fmt::Display for Expr { // ident // ----------------------------------------------------------------------------- -#[derive(Clone, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Ident(pub(crate) String); impl Ident { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b17a18f7c3..6c50177d2c 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -35,7 +35,7 @@ lalrpop_mod!( pub mod ast; mod lex; -mod template_string; +pub mod template_string; pub use crate::diagnostic::Span; pub use ast::{Literal, Program}; diff --git a/src/parser/template_string.rs b/src/parser/template_string.rs index e5d21cac2d..8f55b553c9 100644 --- a/src/parser/template_string.rs +++ b/src/parser/template_string.rs @@ -24,6 +24,7 @@ pub struct TemplateString(pub Vec); impl TemplateString { /// Rewrites the ast for the template string to be a series of string concatenations + #[must_use] pub fn rewrite_to_concatenated_strings(&self) -> Expr { self.0 .iter() @@ -65,6 +66,7 @@ impl TemplateString { /// If the template string is just a single literal string return that string /// as we can just represent it in the ast as a single literal, otherwise return /// None as we will need to rewrite it into an expression. + #[must_use] pub fn as_literal_string(&self) -> Option<&str> { match self.0.as_slice() { [StringSegment::Literal(s, _)] => Some(s), diff --git a/src/path/mod.rs b/src/path/mod.rs index ad3e335f40..27cf62622f 100644 --- a/src/path/mod.rs +++ b/src/path/mod.rs @@ -311,6 +311,7 @@ fn get_target_prefix(path: &str) -> (PathPrefix, &str) { } #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[cfg_attr(any(test, feature = "proptest"), derive(proptest_derive::Arbitrary))] pub enum PathPrefix { Event, Metadata, diff --git a/src/path/owned.rs b/src/path/owned.rs index 7d8a9dbe42..bc7e57e84b 100644 --- a/src/path/owned.rs +++ b/src/path/owned.rs @@ -1,7 +1,11 @@ +use std::fmt::{self, Debug, Display, Formatter, Write}; +use std::str::FromStr; + use once_cell::sync::Lazy; +#[cfg(any(test, feature = "proptest"))] +use proptest::prelude::*; use regex::Regex; use serde::{Deserialize, Serialize}; -use std::fmt::{Debug, Display, Formatter}; use super::PathPrefix; use super::{parse_target_path, parse_value_path, BorrowedSegment, PathParseError, ValuePath}; @@ -117,8 +121,22 @@ impl OwnedValuePath { } } +// OwnedValuePath values must have at least one segment. +#[cfg(any(test, feature = "proptest"))] +impl Arbitrary for OwnedValuePath { + type Parameters = (); + type Strategy = BoxedStrategy; + + fn arbitrary_with((): Self::Parameters) -> Self::Strategy { + prop::collection::vec(any::(), 1..10) + .prop_map(|segments| OwnedValuePath { segments }) + .boxed() + } +} + /// An owned path that contains a target (pointing to either an Event or Metadata) #[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[cfg_attr(any(test, feature = "proptest"), derive(proptest_derive::Arbitrary))] #[serde(try_from = "String", into = "String")] pub struct OwnedTargetPath { pub prefix: PathPrefix, @@ -181,29 +199,46 @@ impl OwnedTargetPath { } impl Display for OwnedTargetPath { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", String::from(self.to_owned())) + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self.prefix { + PathPrefix::Event => write!(f, ".")?, + PathPrefix::Metadata => write!(f, "%")?, + } + Display::fmt(&self.path, f) } } impl Debug for OwnedTargetPath { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { Display::fmt(self, f) } } impl From for String { fn from(target_path: OwnedTargetPath) -> Self { - match target_path.prefix { - PathPrefix::Event => format!(".{}", target_path.path), - PathPrefix::Metadata => format!("%{}", target_path.path), - } + Self::from(&target_path) + } +} + +impl From<&OwnedTargetPath> for String { + fn from(target_path: &OwnedTargetPath) -> Self { + target_path.to_string() } } impl Display for OwnedValuePath { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", String::from(self.clone())) + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", String::from(self)) + } +} + +impl FromStr for OwnedValuePath { + type Err = PathParseError; + + fn from_str(src: &str) -> Result { + parse_value_path(src).map_err(|_| PathParseError::InvalidPathSyntax { + path: src.to_owned(), + }) } } @@ -211,7 +246,15 @@ impl TryFrom for OwnedValuePath { type Error = PathParseError; fn try_from(src: String) -> Result { - parse_value_path(&src).map_err(|_| PathParseError::InvalidPathSyntax { + src.parse() + } +} + +impl FromStr for OwnedTargetPath { + type Err = PathParseError; + + fn from_str(src: &str) -> Result { + parse_target_path(src).map_err(|_| PathParseError::InvalidPathSyntax { path: src.to_owned(), }) } @@ -221,9 +264,7 @@ impl TryFrom for OwnedTargetPath { type Error = PathParseError; fn try_from(src: String) -> Result { - parse_target_path(&src).map_err(|_| PathParseError::InvalidPathSyntax { - path: src.to_owned(), - }) + src.parse() } } @@ -231,7 +272,7 @@ impl TryFrom for OwnedValuePath { type Error = PathParseError; fn try_from(src: KeyString) -> Result { - parse_value_path(&src).map_err(|_| PathParseError::InvalidPathSyntax { path: src.into() }) + src.parse() } } @@ -239,61 +280,64 @@ impl TryFrom for OwnedTargetPath { type Error = PathParseError; fn try_from(src: KeyString) -> Result { - parse_target_path(&src).map_err(|_| PathParseError::InvalidPathSyntax { path: src.into() }) + src.parse() } } impl From for String { fn from(owned: OwnedValuePath) -> Self { - let mut coalesce_i = 0; - owned - .segments - .iter() - .enumerate() - .map(|(i, segment)| match segment { + Self::from(&owned) + } +} + +impl From<&OwnedValuePath> for String { + fn from(owned: &OwnedValuePath) -> Self { + let mut output = String::new(); + for (i, segment) in owned.segments.iter().enumerate() { + match segment { OwnedSegment::Field(field) => { - serialize_field(field.as_ref(), (i != 0).then_some(".")) + serialize_field(&mut output, field.as_ref(), (i != 0).then_some(".")) + } + OwnedSegment::Index(index) => { + write!(output, "[{index}]").expect("Could not write to string") } - OwnedSegment::Index(index) => format!("[{}]", index), OwnedSegment::Coalesce(fields) => { - let mut output = String::new(); + let mut coalesce_i = 0; let (last, fields) = fields.split_last().expect("coalesce must not be empty"); for field in fields { - let field_output = serialize_field( + serialize_field( + &mut output, field.as_ref(), - Some(if coalesce_i == 0 { - if i == 0 { - "(" - } else { - ".(" - } - } else { + Some(if coalesce_i != 0 { "|" + } else if i == 0 { + "(" + } else { + ".(" }), ); - coalesce_i += 1; - output.push_str(&field_output); + coalesce_i = 1; } - output += &serialize_field(last.as_ref(), (coalesce_i != 0).then_some("|")); - output += ")"; - output + serialize_field(&mut output, last.as_ref(), (coalesce_i != 0).then_some("|")); + output.push(')'); } - }) - .collect::>() - .join("") + } + } + output } } -fn serialize_field(field: &str, separator: Option<&str>) -> String { +fn serialize_field(string: &mut String, field: &str, separator: Option<&str>) { // These characters should match the ones from the parser, implemented in `JitLookup` - let needs_quotes = field - .chars() - .any(|c| !matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '0'..='9' | '@')); + let needs_quotes = field.is_empty() + || field + .chars() + .any(|c| !matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '0'..='9' | '@')); - // Allocate enough to fit the field, a `.` and two `"` characters. This + // Reserve enough to fit the field, a `.` and two `"` characters. This // should suffice for the majority of cases when no escape sequence is used. - let separator_len = separator.map(|x| x.len()).unwrap_or(0); - let mut string = String::with_capacity(field.as_bytes().len() + 2 + separator_len); + let separator_len = separator.map_or(0, |x| x.len()); + string.reserve(field.as_bytes().len() + 2 + separator_len); if let Some(separator) = separator { string.push_str(separator); } @@ -306,10 +350,8 @@ fn serialize_field(field: &str, separator: Option<&str>) -> String { string.push(c); } string.push('"'); - string } else { string.push_str(field); - string } } @@ -361,6 +403,24 @@ impl OwnedSegment { } } +// This is almost the same as the automatically-derived implementation, except that we explictly +// restrict the length of the `Coalesce` variant to at least two elements, which is a constraint of +// the textual representation. +#[cfg(any(test, feature = "proptest"))] +impl Arbitrary for OwnedSegment { + type Parameters = (); + type Strategy = BoxedStrategy; + + fn arbitrary_with((): Self::Parameters) -> Self::Strategy { + prop_oneof![ + any::().prop_map(OwnedSegment::Field), + any::().prop_map(OwnedSegment::Index), + prop::collection::vec(any::(), 2..10).prop_map(OwnedSegment::Coalesce), + ] + .boxed() + } +} + impl From> for OwnedSegment { fn from(fields: Vec<&'static str>) -> Self { fields @@ -430,31 +490,33 @@ impl<'a> ValuePath<'a> for &'a OwnedValuePath { static VALID_FIELD: Lazy = Lazy::new(|| Regex::new("^[0-9]*[a-zA-Z_@][0-9a-zA-Z_@]*$").unwrap()); -fn field_to_string(field: &str) -> String { +fn format_field(f: &mut Formatter<'_>, field: &str) -> fmt::Result { // This can eventually just parse the field and see if it's valid, but the // parser is currently lenient in what it accepts so it doesn't catch all cases that // should be quoted let needs_quotes = !VALID_FIELD.is_match(field); if needs_quotes { - format!("\"{}\"", field) + write!(f, "\"{field}\"") } else { - field.to_string() + write!(f, "{field}") } } impl Display for OwnedSegment { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { - OwnedSegment::Index(i) => write!(f, "[{}]", i), - OwnedSegment::Field(field) => write!(f, "{}", field_to_string(field)), - OwnedSegment::Coalesce(v) => write!( - f, - "({})", - v.iter() - .map(|field| field_to_string(field)) - .collect::>() - .join(" | ") - ), + OwnedSegment::Index(i) => write!(f, "[{i}]"), + OwnedSegment::Field(field) => format_field(f, field), + OwnedSegment::Coalesce(v) => { + write!(f, "(")?; + for (i, field) in v.iter().enumerate() { + if i > 0 { + write!(f, " | ")?; + } + format_field(f, field)?; + } + write!(f, ")") + } } } } @@ -496,6 +558,7 @@ impl<'a> Iterator for OwnedSegmentSliceIter<'a> { #[cfg(test)] mod test { + use super::*; use crate::path::parse_value_path; #[test] @@ -546,4 +609,25 @@ mod test { assert_eq!(path, expected.map(|x| x.to_owned())); } } + + fn reparse_thing(thing: T) + where + ::Err: std::fmt::Debug, + { + let text = thing.to_string(); + let thing2: T = text.parse().unwrap(); + assert_eq!(thing, thing2); + } + + proptest::proptest! { + #[test] + fn reparses_valid_value_path(path: OwnedValuePath) { + reparse_thing(path); + } + + #[test] + fn reparses_valid_target_path(path: OwnedTargetPath) { + reparse_thing(path); + } + } } diff --git a/src/stdlib/decode_punycode.rs b/src/stdlib/decode_punycode.rs new file mode 100644 index 0000000000..fd9f0c3446 --- /dev/null +++ b/src/stdlib/decode_punycode.rs @@ -0,0 +1,87 @@ +use crate::compiler::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub struct DecodePunycode; + +impl Function for DecodePunycode { + fn identifier(&self) -> &'static str { + "decode_punycode" + } + + fn parameters(&self) -> &'static [Parameter] { + &[Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }] + } + + fn compile( + &self, + _state: &state::TypeState, + _ctx: &mut FunctionCompileContext, + arguments: ArgumentList, + ) -> Compiled { + let value = arguments.required("value"); + + Ok(DecodePunycodeFn { value }.as_expr()) + } + + fn examples(&self) -> &'static [Example] { + &[ + Example { + title: "punycode string", + source: r#"decode_punycode!("www.xn--caf-dma.com")"#, + result: Ok("www.café.com"), + }, + Example { + title: "ascii string", + source: r#"decode_punycode!("www.cafe.com")"#, + result: Ok("www.cafe.com"), + }, + ] + } +} + +#[derive(Clone, Debug)] +struct DecodePunycodeFn { + value: Box, +} + +impl FunctionExpression for DecodePunycodeFn { + fn resolve(&self, ctx: &mut Context) -> Resolved { + let value = self.value.resolve(ctx)?; + let string = value.try_bytes_utf8_lossy()?; + + let (encoded, result) = idna::domain_to_unicode(&string); + result.map_err(|errors| format!("unable to decode punycode: {errors}"))?; + + Ok(encoded.into()) + } + + fn type_def(&self, _: &state::TypeState) -> TypeDef { + TypeDef::bytes().fallible() + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::value; + + test_function![ + decode_punycode => DecodePunycode; + + demo_string { + args: func_args![value: value!("www.xn--caf-dma.com")], + want: Ok(value!("www.café.com")), + tdef: TypeDef::bytes().fallible(), + } + + ascii_string { + args: func_args![value: value!("www.cafe.com")], + want: Ok(value!("www.cafe.com")), + tdef: TypeDef::bytes().fallible(), + } + ]; +} diff --git a/src/stdlib/encode_punycode.rs b/src/stdlib/encode_punycode.rs new file mode 100644 index 0000000000..da002d0116 --- /dev/null +++ b/src/stdlib/encode_punycode.rs @@ -0,0 +1,98 @@ +use crate::compiler::prelude::*; + +#[derive(Clone, Copy, Debug)] +pub struct EncodePunycode; + +impl Function for EncodePunycode { + fn identifier(&self) -> &'static str { + "encode_punycode" + } + + fn parameters(&self) -> &'static [Parameter] { + &[Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }] + } + + fn compile( + &self, + _state: &state::TypeState, + _ctx: &mut FunctionCompileContext, + arguments: ArgumentList, + ) -> Compiled { + let value = arguments.required("value"); + + Ok(EncodePunycodeFn { value }.as_expr()) + } + + fn examples(&self) -> &'static [Example] { + &[ + Example { + title: "IDN string", + source: r#"encode_punycode!("www.café.com")"#, + result: Ok("www.xn--caf-dma.com"), + }, + Example { + title: "mixed case string", + source: r#"encode_punycode!("www.CAFé.com")"#, + result: Ok("www.xn--caf-dma.com"), + }, + Example { + title: "ascii string", + source: r#"encode_punycode!("www.cafe.com")"#, + result: Ok("www.cafe.com"), + }, + ] + } +} + +#[derive(Clone, Debug)] +struct EncodePunycodeFn { + value: Box, +} + +impl FunctionExpression for EncodePunycodeFn { + fn resolve(&self, ctx: &mut Context) -> Resolved { + let value = self.value.resolve(ctx)?; + let string = value.try_bytes_utf8_lossy()?; + + let encoded = idna::domain_to_ascii(&string) + .map_err(|errors| format!("unable to encode to punycode: {errors}"))?; + + Ok(encoded.into()) + } + + fn type_def(&self, _: &state::TypeState) -> TypeDef { + TypeDef::bytes().fallible() + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::value; + + test_function![ + encode_punycode => EncodePunycode; + + idn_string { + args: func_args![value: value!("www.café.com")], + want: Ok(value!("www.xn--caf-dma.com")), + tdef: TypeDef::bytes().fallible(), + } + + mixed_case { + args: func_args![value: value!("www.CAFé.com")], + want: Ok(value!("www.xn--caf-dma.com")), + tdef: TypeDef::bytes().fallible(), + } + + ascii_string { + args: func_args![value: value!("www.cafe.com")], + want: Ok(value!("www.cafe.com")), + tdef: TypeDef::bytes().fallible(), + } + ]; +} diff --git a/src/stdlib/get_timezone_name.rs b/src/stdlib/get_timezone_name.rs new file mode 100644 index 0000000000..150edc962e --- /dev/null +++ b/src/stdlib/get_timezone_name.rs @@ -0,0 +1,74 @@ +use crate::compiler::prelude::*; +use chrono::Local; +use std::borrow::Cow; + +#[must_use] +pub fn get_name_for_timezone(tz: &TimeZone) -> Cow<'_, str> { + match tz { + TimeZone::Named(tz) => tz.name().into(), + TimeZone::Local => iana_time_zone::get_timezone() + .unwrap_or_else(|_| Local::now().offset().to_string()) + .into(), + } +} + +#[allow(clippy::unnecessary_wraps)] +fn get_timezone_name(ctx: &mut Context) -> Resolved { + Ok(get_name_for_timezone(ctx.timezone()).into()) +} + +#[derive(Clone, Copy, Debug)] +pub struct GetTimezoneName; + +impl Function for GetTimezoneName { + fn identifier(&self) -> &'static str { + "get_timezone_name" + } + + fn examples(&self) -> &'static [Example] { + &[Example { + title: "Get the VRL timezone name, or for 'local' the local timezone name or offset (e.g., -05:00)", + source: r#"get_timezone_name!() != """#, + result: Ok("true"), + }] + } + + fn compile( + &self, + _state: &TypeState, + _ctx: &mut FunctionCompileContext, + _: ArgumentList, + ) -> Compiled { + Ok(GetTimezoneNameFn.as_expr()) + } +} + +#[derive(Debug, Clone)] +struct GetTimezoneNameFn; + +impl FunctionExpression for GetTimezoneNameFn { + fn resolve(&self, ctx: &mut Context) -> Resolved { + get_timezone_name(ctx) + } + + fn type_def(&self, _: &TypeState) -> TypeDef { + TypeDef::bytes().fallible() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::value; + + test_function![ + get_hostname => GetTimezoneName; + + // the test harness always initializes the VRL timezone to UTC + utc { + args: func_args![], + want: Ok(value!(get_name_for_timezone(&TimeZone::Named(chrono_tz::Tz::UTC)))), + tdef: TypeDef::bytes().fallible(), + } + ]; +} diff --git a/src/stdlib/mod.rs b/src/stdlib/mod.rs index 00c9af563e..b4e5928330 100644 --- a/src/stdlib/mod.rs +++ b/src/stdlib/mod.rs @@ -53,6 +53,7 @@ cfg_if::cfg_if! { mod decode_gzip; mod decode_mime_q; mod decode_percent; + mod decode_punycode; mod decode_snappy; mod decode_zlib; mod decode_zstd; @@ -66,6 +67,7 @@ cfg_if::cfg_if! { mod encode_key_value; mod encode_logfmt; mod encode_percent; + mod encode_punycode; mod encode_snappy; mod encode_zlib; mod encode_zstd; @@ -85,6 +87,7 @@ cfg_if::cfg_if! { mod get; mod get_env_var; mod get_hostname; + mod get_timezone_name; mod hmac; mod includes; mod integer; @@ -164,6 +167,7 @@ cfg_if::cfg_if! { mod parse_common_log; mod parse_csv; mod parse_duration; + mod parse_etld; mod parse_float; mod parse_glog; mod parse_grok; @@ -194,6 +198,7 @@ cfg_if::cfg_if! { mod redact; mod remove; mod replace; + mod replace_with; mod reverse_dns; mod round; mod seahash; @@ -248,6 +253,7 @@ cfg_if::cfg_if! { pub use decode_gzip::DecodeGzip; pub use decode_mime_q::DecodeMimeQ; pub use decode_percent::DecodePercent; + pub use decode_punycode::DecodePunycode; pub use decode_snappy::DecodeSnappy; pub use decode_zlib::DecodeZlib; pub use decode_zstd::DecodeZstd; @@ -261,6 +267,7 @@ cfg_if::cfg_if! { pub use encode_key_value::EncodeKeyValue; pub use encode_logfmt::EncodeLogfmt; pub use encode_percent::EncodePercent; + pub use encode_punycode::EncodePunycode; pub use encode_snappy::EncodeSnappy; pub use encode_zlib::EncodeZlib; pub use encode_zstd::EncodeZstd; @@ -281,6 +288,8 @@ cfg_if::cfg_if! { pub use get::Get; pub use get_env_var::GetEnvVar; pub use get_hostname::GetHostname; + pub use get_timezone_name::GetTimezoneName; + pub use get_timezone_name::get_name_for_timezone; pub use includes::Includes; pub use integer::Integer; pub use ip_aton::IpAton; @@ -356,6 +365,7 @@ cfg_if::cfg_if! { pub use parse_csv::ParseCsv; pub use parse_duration::ParseDuration; pub use parse_float::ParseFloat; + pub use parse_etld::ParseEtld; pub use parse_glog::ParseGlog; pub use parse_grok::ParseGrok; pub use parse_groks::ParseGroks; @@ -386,6 +396,7 @@ cfg_if::cfg_if! { pub use redact::Redact; pub use remove::Remove; pub use replace::Replace; + pub use replace_with::ReplaceWith; pub use reverse_dns::ReverseDns; pub use round::Round; pub use set::Set; @@ -444,6 +455,7 @@ pub fn all() -> Vec> { Box::new(DecodeBase64), Box::new(DecodeGzip), Box::new(DecodePercent), + Box::new(DecodePunycode), Box::new(DecodeMimeQ), Box::new(DecodeSnappy), Box::new(DecodeZlib), @@ -458,6 +470,7 @@ pub fn all() -> Vec> { Box::new(EncodeKeyValue), Box::new(EncodeLogfmt), Box::new(EncodePercent), + Box::new(EncodePunycode), Box::new(EncodeSnappy), Box::new(EncodeZlib), Box::new(EncodeZstd), @@ -477,6 +490,7 @@ pub fn all() -> Vec> { Box::new(Get), Box::new(GetEnvVar), Box::new(GetHostname), + Box::new(GetTimezoneName), Box::new(Hmac), Box::new(Includes), Box::new(Integer), @@ -558,6 +572,7 @@ pub fn all() -> Vec> { Box::new(ParseCsv), Box::new(ParseDuration), Box::new(ParseFloat), + Box::new(ParseEtld), Box::new(ParseGlog), Box::new(ParseGrok), Box::new(ParseGroks), @@ -587,6 +602,7 @@ pub fn all() -> Vec> { Box::new(Redact), Box::new(Remove), Box::new(Replace), + Box::new(ReplaceWith), Box::new(ReverseDns), Box::new(Round), Box::new(Seahash), diff --git a/src/stdlib/parse_etld.rs b/src/stdlib/parse_etld.rs new file mode 100644 index 0000000000..7f566db041 --- /dev/null +++ b/src/stdlib/parse_etld.rs @@ -0,0 +1,240 @@ +use crate::compiler::prelude::*; +use std::collections::BTreeMap; + +#[derive(Clone, Copy, Debug)] +pub struct ParseEtld; + +impl Function for ParseEtld { + fn identifier(&self) -> &'static str { + "parse_etld" + } + + fn parameters(&self) -> &'static [Parameter] { + &[ + Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }, + Parameter { + keyword: "plus_parts", + kind: kind::INTEGER, + required: false, + }, + ] + } + + fn examples(&self) -> &'static [Example] { + &[ + Example { + title: "parse etld", + source: r#"parse_etld!("vector.dev")"#, + result: Ok(indoc! {r#" + { + "etld": "dev", + "etld_plus": "dev", + "known_suffix": true + } + "#}), + }, + Example { + title: "parse etld with plus parts", + source: r#"parse_etld!("vector.dev", plus_parts: 1)"#, + result: Ok(indoc! {r#" + { + "etld": "dev", + "etld_plus": "vector.dev", + "known_suffix": true + } + "#}), + }, + Example { + title: "parse etld with unknown suffix", + source: r#"parse_etld!("vecor.unknowndev")"#, + result: Ok(indoc! {r#" + { + "etld": "unknowndev", + "etld_plus": "unknowndev", + "known_suffix": false + } + "#}), + }, + ] + } + + fn compile( + &self, + _state: &state::TypeState, + _ctx: &mut FunctionCompileContext, + arguments: ArgumentList, + ) -> Compiled { + let value = arguments.required("value"); + let plus_parts = arguments.optional("plus_parts").unwrap_or_else(|| expr!(0)); + + Ok(ParseEtldFn { value, plus_parts }.as_expr()) + } +} + +#[derive(Debug, Clone)] +struct ParseEtldFn { + value: Box, + plus_parts: Box, +} + +impl FunctionExpression for ParseEtldFn { + fn resolve(&self, ctx: &mut Context) -> Resolved { + let value = self.value.resolve(ctx)?; + let string = value.try_bytes_utf8_lossy()?; + + let plus_parts = match self.plus_parts.resolve(ctx)?.try_integer()? { + x if x < 0 => 0, + x => x as usize, + }; + + let etld = psl::suffix(string.as_bytes()) + .ok_or(format!("unable to determine eTLD for {string}"))?; + let etld_string = core::str::from_utf8(etld.as_bytes()) + .map_err(|err| format!("could not convert eTLD to UTF8 {err}"))?; + + let etld_parts_count = etld_string.chars().filter(|c| *c == '.').count() + 1; + let etld_plus_parts: Vec<&str> = string + .rsplit('.') + .take(etld_parts_count + plus_parts) + .collect(); + + let etld_plus = etld_plus_parts + .into_iter() + .rev() + .collect::>() + .join("."); + + let mut map = BTreeMap::<&str, Value>::new(); + + map.insert("etld", etld_string.to_owned().into()); + map.insert("etld_plus", etld_plus.into()); + map.insert("known_suffix", etld.is_known().into()); + + Ok(map + .into_iter() + .map(|(k, v)| (k.to_owned(), v)) + .collect::()) + } + + fn type_def(&self, _: &state::TypeState) -> TypeDef { + TypeDef::object(inner_kind()).fallible() + } +} + +fn inner_kind() -> BTreeMap { + BTreeMap::from([ + ("etld".into(), Kind::bytes()), + ("etld_plus".into(), Kind::bytes()), + ("known_suffix".into(), Kind::boolean()), + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::value; + + test_function![ + parse_etld => ParseEtld; + + naive { + args: func_args![value: value!("vector.dev")], + want: Ok(value!({ + etld: "dev", + etld_plus: "dev", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + naive_plus_one { + args: func_args![value: value!("vector.dev"), plus_parts: 1], + want: Ok(value!({ + etld: "dev", + etld_plus: "vector.dev", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + psl { + args: func_args![value: value!("sussex.ac.uk")], + want: Ok(value!({ + etld: "ac.uk", + etld_plus: "ac.uk", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + psl_plus_one { + args: func_args![value: value!("sussex.ac.uk"), plus_parts: 1], + want: Ok(value!({ + etld: "ac.uk", + etld_plus: "sussex.ac.uk", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + short_plus { + args: func_args![value: value!("vector.dev"), plus_parts: 10], + want: Ok(value!({ + etld: "dev", + etld_plus: "vector.dev", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + long_plus { + args: func_args![value: value!("www.long.plus.test.vector.dev"), plus_parts: 4], + want: Ok(value!({ + etld: "dev", + etld_plus: "long.plus.test.vector.dev", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + unknown_tld { + args: func_args![value: value!("vector.unknowndev")], + want: Ok(value!({ + etld: "unknowndev", + etld_plus: "unknowndev", + known_suffix: false, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + utf8 { + args: func_args![value: value!("www.食狮.中国")], + want: Ok(value!({ + etld: "中国", + etld_plus: "中国", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + utf8_plus_one { + args: func_args![value: value!("www.食狮.中国"), plus_parts: 1], + want: Ok(value!({ + etld: "中国", + etld_plus: "食狮.中国", + known_suffix: true, + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + empty_host { + args: func_args![value: value!("")], + want: Err("unable to determine eTLD for "), + tdef: TypeDef::object(inner_kind()).fallible(), + } + ]; +} diff --git a/src/stdlib/parse_groks.rs b/src/stdlib/parse_groks.rs index fbe360bd72..c7b652ec37 100644 --- a/src/stdlib/parse_groks.rs +++ b/src/stdlib/parse_groks.rs @@ -65,6 +65,8 @@ mod non_wasm { #[allow(clippy::wildcard_imports)] #[cfg(not(target_arch = "wasm32"))] use non_wasm::*; +#[cfg(not(target_arch = "wasm32"))] +use std::{fs::File, io::BufReader, path::Path}; #[derive(Clone, Copy, Debug)] pub struct ParseGroks; @@ -91,6 +93,11 @@ impl Function for ParseGroks { kind: kind::OBJECT, required: false, }, + Parameter { + keyword: "alias_sources", + kind: kind::ARRAY, + required: false, + }, ] } @@ -138,38 +145,87 @@ impl Function for ParseGroks { .into_iter() .map(|expr| { let pattern = expr + .clone() .resolve_constant(state) .ok_or(function::Error::ExpectedStaticExpression { keyword: "patterns", - expr, + expr: expr.clone(), })? .try_bytes_utf8_lossy() - .expect("grok pattern not bytes") + .map_err(|_| function::Error::InvalidArgument { + keyword: "patterns", + value: format!("{expr:?}").into(), + error: "grok pattern should be a string", + })? .into_owned(); Ok(pattern) }) .collect::, function::Error>>()?; - let aliases = arguments + let mut aliases = arguments .optional_object("aliases")? .unwrap_or_default() .into_iter() .map(|(key, expr)| { let alias = expr + .clone() .resolve_constant(state) .ok_or(function::Error::ExpectedStaticExpression { keyword: "aliases", - expr, - }) - .map(|e| { - e.try_bytes_utf8_lossy() - .expect("should be a string") - .into_owned() - })?; + expr: expr.clone(), + })? + .try_bytes_utf8_lossy() + .map_err(|_| function::Error::InvalidArgument { + keyword: "aliases", + value: format!("{expr:?}").into(), + error: "alias pattern should be a string", + })? + .into_owned(); Ok((key, alias)) }) .collect::, function::Error>>()?; + let alias_sources = arguments + .optional_array("alias_sources")? + .unwrap_or_default() + .into_iter() + .map(|expr| { + let path = expr + .clone() + .resolve_constant(state) + .ok_or(function::Error::ExpectedStaticExpression { + keyword: "alias_sources", + expr: expr.clone(), + })? + .try_bytes_utf8_lossy() + .map_err(|_| function::Error::InvalidArgument { + keyword: "alias_sources", + value: format!("{expr:?}").into(), + error: "alias source should be a string", + })? + .into_owned(); + Ok(path) + }) + .collect::, function::Error>>()?; + + for src in alias_sources { + let path = Path::new(&src); + let file = File::open(path).map_err(|_| function::Error::InvalidArgument { + keyword: "alias_sources", + value: format!("{path:?}").into(), + error: "Unable to open alias source file", + })?; + let reader = BufReader::new(file); + let mut src_aliases = + serde_json::from_reader(reader).map_err(|_| function::Error::InvalidArgument { + keyword: "alias_sources", + value: format!("{path:?}").into(), + error: "Unable to read alias source", + })?; + + aliases.append(&mut src_aliases); + } + // we use a datadog library here because it is a superset of grok let grok_rules = crate::datadog_grok::parse_grok_rules::parse_grok_rules( &patterns, aliases, @@ -226,6 +282,16 @@ mod test { tdef: TypeDef::object(Collection::any()).fallible(), } + error3 { + args: func_args![ value: "2020-10-02T23:22:12.223222Z info Hello world", + patterns: vec!["%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}"], + aliases: value!({ + "TEST": 3 + })], + want: Err("invalid argument"), + tdef: TypeDef::object(Collection::any()).fallible(), + } + parsed { args: func_args![ value: "2020-10-02T23:22:12.223222Z info Hello world", patterns: vec!["%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}"]], @@ -270,6 +336,31 @@ mod test { tdef: TypeDef::object(Collection::any()).fallible(), } + presence_of_alias_sources_argument { + args: func_args![ + value: r##"2020-10-02T23:22:12.223222Z info 200 hello world"##, + patterns: Value::Array(vec![ + "%{common_prefix} %{_status} %{_message}".into(), + "%{common_prefix} %{_message}".into(), + ]), + aliases: value!({ + "common_prefix": "%{_timestamp} %{_loglevel}", + "_timestamp": "%{TIMESTAMP_ISO8601:timestamp}", + "_loglevel": "%{LOGLEVEL:level}", + "_status": "%{POSINT:status}", + "_message": "%{GREEDYDATA:message}" + }), + alias_sources: Value::Array(vec![]), + ], + want: Ok(Value::from(btreemap! { + "timestamp" => "2020-10-02T23:22:12.223222Z", + "level" => "info", + "status" => "200", + "message" => "hello world" + })), + tdef: TypeDef::object(Collection::any()).fallible(), + } + multiple_patterns_and_aliases_second_pattern_matches { args: func_args![ value: r##"2020-10-02T23:22:12.223222Z info hello world"##, diff --git a/src/stdlib/parse_klog.rs b/src/stdlib/parse_klog.rs index e73a2e8c7c..8e03e26b2a 100644 --- a/src/stdlib/parse_klog.rs +++ b/src/stdlib/parse_klog.rs @@ -88,7 +88,7 @@ impl Function for ParseKlog { "level": "info", "line": 70, "message": "hello from klog", - "timestamp": "2023-05-05T17:59:40.692994Z" + "timestamp": "2024-05-05T17:59:40.692994Z" }"#}), }] } diff --git a/src/stdlib/parse_linux_authorization.rs b/src/stdlib/parse_linux_authorization.rs index 290ee3d5bc..d2eadf54d0 100644 --- a/src/stdlib/parse_linux_authorization.rs +++ b/src/stdlib/parse_linux_authorization.rs @@ -27,7 +27,7 @@ impl Function for ParseLinuxAuthorization { "hostname": "localhost", "message": "Accepted publickey for eng from 10.1.1.1 port 8888 ssh2: RSA SHA256:foobar", "procid": 1111, - "timestamp": "2023-03-23T01:49:58Z" + "timestamp": "2024-03-23T01:49:58Z" }"#}), }] } diff --git a/src/stdlib/parse_url.rs b/src/stdlib/parse_url.rs index d70746604e..790ae271a8 100644 --- a/src/stdlib/parse_url.rs +++ b/src/stdlib/parse_url.rs @@ -208,5 +208,35 @@ mod tests { })), tdef: TypeDef::object(inner_kind()).fallible(), } + + punycode { + args: func_args![value: value!("https://www.café.com")], + want: Ok(value!({ + fragment: (), + host: "www.xn--caf-dma.com", + password: "", + path: "/", + port: (), + query: {}, + scheme: "https", + username: "", + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } + + punycode_mixed_case { + args: func_args![value: value!("https://www.CAFé.com")], + want: Ok(value!({ + fragment: (), + host: "www.xn--caf-dma.com", + password: "", + path: "/", + port: (), + query: {}, + scheme: "https", + username: "", + })), + tdef: TypeDef::object(inner_kind()).fallible(), + } ]; } diff --git a/src/stdlib/replace_with.rs b/src/stdlib/replace_with.rs new file mode 100644 index 0000000000..1c9efb2ac7 --- /dev/null +++ b/src/stdlib/replace_with.rs @@ -0,0 +1,256 @@ +use std::collections::BTreeMap; + +use regex::{CaptureMatches, CaptureNames, Captures, Regex}; + +use crate::compiler::prelude::*; + +fn replace_with( + value: Value, + pattern: &Regex, + count: Value, + ctx: &mut Context, + runner: closure::Runner, +) -> Resolved +where + T: Fn(&mut Context) -> Result, +{ + let haystack = value.try_bytes_utf8_lossy()?; + let count = match count.try_integer()? { + i if i > 0 => i as usize, + i if i < 0 => 0, + // this is when i == 0 + _ => return Ok(value), + }; + let captures = pattern.captures_iter(&haystack); + make_replacement( + captures, + &haystack, + count, + pattern.capture_names(), + ctx, + runner, + ) +} + +fn make_replacement( + caps: CaptureMatches, + haystack: &str, + count: usize, + capture_names: CaptureNames, + ctx: &mut Context, + runner: closure::Runner, +) -> Resolved +where + T: Fn(&mut Context) -> Result, +{ + // possible optimization: peek at first capture, if none return the original value. + let mut replaced = String::with_capacity(haystack.len()); + let limit = if count == 0 { usize::MAX } else { count - 1 }; + let mut last_match = 0; + // we loop over the matches ourselves instead of calling Regex::replacen, so that we can + // handle errors. This is however based on the implementation of Regex::replacen + for (idx, captures) in caps.enumerate() { + // Safe to unrap because the 0th index always includes the full match. + let m = captures.get(0).unwrap(); // full match + + let mut value = captures_to_value(&captures, capture_names.clone()); + runner.map_value(ctx, &mut value)?; + let replacement = value.try_bytes_utf8_lossy()?; + + replaced.push_str(&haystack[last_match..m.start()]); + replaced.push_str(&replacement); + last_match = m.end(); + if idx >= limit { + break; + } + } + // add the final component + replaced.push_str(&haystack[last_match..]); + Ok(replaced.into()) +} + +const STRING_NAME: &str = "string"; +const CAPTURES_NAME: &str = "captures"; + +fn captures_to_value(captures: &Captures, capture_names: CaptureNames) -> Value { + let mut object: ObjectMap = BTreeMap::new(); + + // The full match, named "string" + object.insert(STRING_NAME.into(), captures.get(0).unwrap().as_str().into()); + // The length includes the total match, so subtract 1 + let mut capture_groups: Vec = Vec::with_capacity(captures.len() - 1); + + // We skip the first entry, because it is for the full match, which we have already + // extracted + for (idx, name) in capture_names.enumerate().skip(1) { + let value: Value = if let Some(group) = captures.get(idx) { + group.as_str().into() + } else { + Value::Null + }; + if let Some(name) = name { + object.insert(name.into(), value.clone()); + } + capture_groups.push(value); + } + + object.insert(CAPTURES_NAME.into(), capture_groups.into()); + + object.into() +} + +#[derive(Clone, Copy, Debug)] +pub struct ReplaceWith; + +impl Function for ReplaceWith { + fn identifier(&self) -> &'static str { + "replace_with" + } + + fn parameters(&self) -> &'static [Parameter] { + &[ + Parameter { + keyword: "value", + kind: kind::BYTES, + required: true, + }, + Parameter { + keyword: "pattern", + kind: kind::REGEX, + required: true, + }, + Parameter { + keyword: "count", + kind: kind::INTEGER, + required: false, + }, + ] + } + + fn examples(&self) -> &'static [Example] { + &[ + Example { + title: "double replacement", + source: r#"replace_with("foobar", r'o|a') -> |m| { m.string + m.string }"#, + result: Ok("foooobaar"), + }, + Example { + title: "replace count", + source: r#"replace_with("foobar", r'o|a', count: 1) -> |m| { m.string + m.string }"#, + result: Ok("fooobar"), + }, + Example { + title: "replace with capture group", + source: r#"replace_with("foo123bar", r'foo(\d+)bar') -> |m| { x = m.captures[0]; "x={{x}}" }"#, + result: Ok(r#"x=123"#), + }, + Example { + title: "process capture group", + source: r#"replace_with(s'Got message: {"msg": "b"}', r'message: (\{.*\})') -> |m| { to_string!(parse_json!(m.captures[0]).msg) }"#, + result: Ok("Got b"), + }, + Example { + title: "Optional capture group", + source: r#"replace_with("foobar", r'bar( of gold)?') -> |m| { if m.captures[1] == null { "baz" } else { "rich" } }"#, + result: Ok("foobaz"), + }, + Example { + title: "Named capture group", + source: r#"replace_with("foo123bar", r'foo(?P\d+)bar') -> |m| { x = to_int!(m.num); to_string(x+ 1) }"#, //to_string(to_int!(m.named.num) + 1) }"#, + result: Ok("\"124\""), + }, + ] + } + + fn compile( + &self, + _state: &state::TypeState, + _ctx: &mut FunctionCompileContext, + arguments: ArgumentList, + ) -> Compiled { + let value = arguments.required("value"); + let pattern = arguments.required("pattern"); + let count = arguments.optional("count").unwrap_or(expr!(-1)); + + let closure = arguments.required_closure()?; + + Ok(ReplaceWithFn { + value, + pattern, + count, + closure, + } + .as_expr()) + } + + fn closure(&self) -> Option { + use closure::{Definition, Input, Output, Variable, VariableKind}; + + let match_type = Collection::from_parts( + BTreeMap::from([ + (STRING_NAME.into(), Kind::bytes()), + ( + CAPTURES_NAME.into(), + Kind::array(Collection::from_unknown(Kind::bytes().or_null())), + ), + ]), + Kind::bytes().or_null(), + ); + + Some(Definition { + inputs: vec![Input { + parameter_keyword: "value", + kind: Kind::bytes(), + variables: vec![ + Variable { + kind: VariableKind::Exact(Kind::object(match_type)), + }, + ], + output: Output::Kind(Kind::bytes()), + example: Example { + title: "replace with hash", + source: r#"replace_with("received email from a@example.com", pattern: r'\w+@\w+\.\w+') -> |match| { sha2(match.string) }"#, + result: Ok("received email from 896bdca840c9304a5d0bdbeacc4ef359e3093f80c9777c9967e31ba0ff99ed58"), + }, + }], + is_iterator: false, + }) + } +} + +#[derive(Debug, Clone)] +struct ReplaceWithFn { + value: Box, + pattern: Box, + count: Box, + closure: FunctionClosure, +} + +impl FunctionExpression for ReplaceWithFn { + fn resolve(&self, ctx: &mut Context) -> ExpressionResult { + let value = self.value.resolve(ctx)?; + let pattern = self.pattern.resolve(ctx)?; + let pattern = pattern + .as_regex() + .ok_or_else(|| ExpressionError::from("failed to resolve regex"))?; + for name in pattern.capture_names().flatten() { + if name == STRING_NAME || name == CAPTURES_NAME { + return Err(ExpressionError::from( + r#"Capture group cannot be named "string" or "captures""#, + )); + } + } + let count = self.count.resolve(ctx)?; + let FunctionClosure { + variables, block, .. + } = &self.closure; + + let runner = closure::Runner::new(variables, |ctx| block.resolve(ctx)); + + replace_with(value, pattern, count, ctx, runner) + } + + fn type_def(&self, _: &state::TypeState) -> TypeDef { + TypeDef::bytes().infallible() + } +} diff --git a/src/test/mod.rs b/src/test/mod.rs index 2b5fc7151b..11886c96da 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -136,7 +136,11 @@ pub fn run_tests( warnings, config: _, }) => { - if warnings.is_empty() { + warnings_count += warnings.len(); + + if test.check_diagnostics { + process_compilation_diagnostics(&test, cfg, warnings, compile_timing_fmt) + } else if warnings.is_empty() { let run_start = Instant::now(); finalize_config(config_metadata); @@ -154,8 +158,13 @@ pub fn run_tests( process_result(result, &mut test, cfg, timings) } else { - warnings_count += warnings.len(); - process_compilation_diagnostics(&test, cfg, warnings, compile_timing_fmt) + println!("{} (diagnostics)", Colour::Red.bold().paint("FAILED")); + if cfg.verbose { + let formatter = Formatter::new(&test.source, warnings); + println!("{formatter}"); + } + // mark as failure, did not expect any warnings + true } } Err(diagnostics) => { diff --git a/src/test/test.rs b/src/test/test.rs index ee2fe619b4..95327ce9b8 100644 --- a/src/test/test.rs +++ b/src/test/test.rs @@ -16,6 +16,7 @@ pub struct Test { pub result: String, pub result_approx: bool, pub skip: bool, + pub check_diagnostics: bool, // paths set to read-only pub read_only_paths: Vec<(OwnedTargetPath, bool)>, } @@ -134,6 +135,7 @@ impl Test { result, result_approx, skip: content.starts_with("# SKIP"), + check_diagnostics: content.starts_with("# DIAGNOSTICS"), read_only_paths, } } @@ -154,6 +156,7 @@ impl Test { result, result_approx: false, skip: false, + check_diagnostics: false, read_only_paths: vec![], } } diff --git a/src/value/keystring.rs b/src/value/keystring.rs index b567475ba9..2615068a22 100644 --- a/src/value/keystring.rs +++ b/src/value/keystring.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; /// The key type value. This is a simple zero-overhead wrapper set up to make it explicit that /// object keys are read-only and their underlying type is opaque and may change for efficiency. #[derive(Clone, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[cfg_attr(any(test, feature = "proptest"), derive(proptest_derive::Arbitrary))] #[serde(transparent)] pub struct KeyString(String);