feat(stdlib): add punycode encoding functions

Cargo.toml

@@ -50,7 +50,7 @@ proptest = ["dep:proptest", "dep:proptest-derive"]
 test = ["string_path"]
 
 # All stdlib functions
-stdlib = ["compiler", "core", "datadog", "dep:aes", "dep:chacha20poly1305", "dep:crypto_secretbox", "dep:ctr", "dep:cbc", "dep:cfb-mode", "dep:ofb", "dep:base16", "dep:nom", "dep:strip-ansi-escapes", "dep:utf8-width", "dep:hex", "dep:seahash", "dep:syslog_loose", "dep:hostname", "dep:zstd", "dep:quoted_printable", "dep:once_cell", "dep:base64", "dep:uuid", "dep:percent-encoding", "dep:uaparser", "dep:rust_decimal", "dep:indexmap", "dep:flate2", "dep:charset", "dep:data-encoding", "dep:hmac", "dep:sha-1", "dep:cidr-utils", "dep:sha-2", "dep:md-5", "dep:url", "dep:woothee", "dep:csv", "dep:roxmltree", "dep:rand", "dep:dns-lookup", "dep:sha-3", "dep:grok", "dep:community-id", "dep:snap"]
+stdlib = ["compiler", "core", "datadog", "dep:aes", "dep:chacha20poly1305", "dep:crypto_secretbox", "dep:ctr", "dep:cbc", "dep:cfb-mode", "dep:ofb", "dep:base16", "dep:nom", "dep:strip-ansi-escapes", "dep:utf8-width", "dep:hex", "dep:seahash", "dep:syslog_loose", "dep:hostname", "dep:zstd", "dep:quoted_printable", "dep:once_cell", "dep:base64", "dep:uuid", "dep:percent-encoding", "dep:uaparser", "dep:rust_decimal", "dep:indexmap", "dep:flate2", "dep:charset", "dep:data-encoding", "dep:hmac", "dep:sha-1", "dep:cidr-utils", "dep:sha-2", "dep:md-5", "dep:url", "dep:woothee", "dep:csv", "dep:roxmltree", "dep:rand", "dep:dns-lookup", "dep:sha-3", "dep:grok", "dep:community-id", "dep:snap", "dep:idna"]
 
 [dependencies]
 cfg-if = "1.0.0"
@@ -74,6 +74,7 @@ exitcode = {version = "1", optional = true }
 flate2 = { version = "1.0.28", default-features = false, features = ["default"], optional = true }
 hex = { version = "0.4", optional = true }
 hmac = { version = "0.12.1", optional = true }
+idna = { version = "0.5", optional = true }
 iana-time-zone = "0.1.59"
 indexmap = { version = "~2.2.2", default-features = false, features = ["std"], optional = true}
 indoc = {version = "2.0.4", optional = true }

changelog.d/672.feature.md

@@ -0,0 +1 @@
+Added `encode_punycode` and `decode_punycode` functions

src/stdlib/decode_punycode.rs

@@ -0,0 +1,87 @@
+use crate::compiler::prelude::*;
+
+#[derive(Clone, Copy, Debug)]
+pub struct DecodePunycode;
+
+impl Function for DecodePunycode {
+    fn identifier(&self) -> &'static str {
+        "decode_punycode"
+    }
+
+    fn parameters(&self) -> &'static [Parameter] {
+        &[Parameter {
+            keyword: "value",
+            kind: kind::BYTES,
+            required: true,
+        }]
+    }
+
+    fn compile(
+        &self,
+        _state: &state::TypeState,
+        _ctx: &mut FunctionCompileContext,
+        arguments: ArgumentList,
+    ) -> Compiled {
+        let value = arguments.required("value");
+
+        Ok(DecodePunycodeFn { value }.as_expr())
+    }
+
+    fn examples(&self) -> &'static [Example] {
+        &[
+            Example {
+                title: "punycode string",
+                source: r#"decode_punycode("www.xn--caf-dma.com")"#,
+                result: Ok("www.café.com"),
+            },
+            Example {
+                title: "ascii string",
+                source: r#"decode_punycode("www.cafe.com")"#,
+                result: Ok("www.cafe.com"),
+            },
+        ]
+    }
+}
+
+#[derive(Clone, Debug)]
+struct DecodePunycodeFn {
+    value: Box<dyn Expression>,
+}
+
+impl FunctionExpression for DecodePunycodeFn {
+    fn resolve(&self, ctx: &mut Context) -> Resolved {
+        let value = self.value.resolve(ctx)?;
+        let string = value.try_bytes_utf8_lossy()?;
+
+        let (encoded, result) = idna::domain_to_unicode(&string);
+        result.map_err(|err| format!("unable to decode punycode: {err}"))?;
+
+        Ok(encoded.into())
+    }
+
+    fn type_def(&self, _: &state::TypeState) -> TypeDef {
+        TypeDef::bytes().infallible()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::value;
+
+    test_function![
+        decode_punycode => DecodePunycode;
+
+        demo_string {
+            args: func_args![value: value!("www.xn--caf-dma.com")],
+            want: Ok(value!("www.café.com")),
+            tdef: TypeDef::bytes().infallible(),
+        }
+
+        ascii_string {
+            args: func_args![value: value!("www.cafe.com")],
+            want: Ok(value!("www.cafe.com")),
+            tdef: TypeDef::bytes().infallible(),
+        }
+    ];
+}

src/stdlib/encode_punycode.rs

@@ -0,0 +1,98 @@
+use crate::compiler::prelude::*;
+
+#[derive(Clone, Copy, Debug)]
+pub struct EncodePunycode;
+
+impl Function for EncodePunycode {
+    fn identifier(&self) -> &'static str {
+        "encode_punycode"
+    }
+
+    fn parameters(&self) -> &'static [Parameter] {
+        &[Parameter {
+            keyword: "value",
+            kind: kind::BYTES,
+            required: true,
+        }]
+    }
+
+    fn compile(
+        &self,
+        _state: &state::TypeState,
+        _ctx: &mut FunctionCompileContext,
+        arguments: ArgumentList,
+    ) -> Compiled {
+        let value = arguments.required("value");
+
+        Ok(EncodePunycodeFn { value }.as_expr())
+    }
+
+    fn examples(&self) -> &'static [Example] {
+        &[
+            Example {
+                title: "IDN string",
+                source: r#"encode_punycode("www.café.com")"#,
+                result: Ok("www.xn--caf-dma.com"),
+            },
+            Example {
+                title: "mixed case string",
+                source: r#"encode_punycode("www.CAFé.com")"#,
+                result: Ok("www.xn--caf-dma.com"),
+            },
+            Example {
+                title: "ascii string",
+                source: r#"encode_punycode("www.cafe.com")"#,
+                result: Ok("www.cafe.com"),
+            },
+        ]
+    }
+}
+
+#[derive(Clone, Debug)]
+struct EncodePunycodeFn {
+    value: Box<dyn Expression>,
+}
+
+impl FunctionExpression for EncodePunycodeFn {
+    fn resolve(&self, ctx: &mut Context) -> Resolved {
+        let value = self.value.resolve(ctx)?;
+        let string = value.try_bytes_utf8_lossy()?;
+
+        let encoded = idna::domain_to_ascii(&string)
+            .map_err(|err| format!("unable to encode to punycode: {err}"))?;
+
+        Ok(encoded.into())
+    }
+
+    fn type_def(&self, _: &state::TypeState) -> TypeDef {
+        TypeDef::bytes().infallible()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::value;
+
+    test_function![
+        encode_punycode => EncodePunycode;
+
+        idn_string {
+            args: func_args![value: value!("www.café.com")],
+            want: Ok(value!("www.xn--caf-dma.com")),
+            tdef: TypeDef::bytes().infallible(),
+        }
+
+        mixed_case {
+            args: func_args![value: value!("www.CAFé.com")],
+            want: Ok(value!("www.xn--caf-dma.com")),
+            tdef: TypeDef::bytes().infallible(),
+        }
+
+        ascii_string {
+            args: func_args![value: value!("www.cafe.com")],
+            want: Ok(value!("www.cafe.com")),
+            tdef: TypeDef::bytes().infallible(),
+        }
+    ];
+}

src/stdlib/mod.rs

@@ -53,6 +53,7 @@ cfg_if::cfg_if! {
         mod decode_gzip;
         mod decode_mime_q;
         mod decode_percent;
+        mod decode_punycode;
         mod decode_snappy;
         mod decode_zlib;
         mod decode_zstd;
@@ -66,6 +67,7 @@ cfg_if::cfg_if! {
         mod encode_key_value;
         mod encode_logfmt;
         mod encode_percent;
+        mod encode_punycode;
         mod encode_snappy;
         mod encode_zlib;
         mod encode_zstd;
@@ -219,6 +221,7 @@ cfg_if::cfg_if! {
         pub use decode_gzip::DecodeGzip;
         pub use decode_mime_q::DecodeMimeQ;
         pub use decode_percent::DecodePercent;
+        pub use decode_punycode::DecodePunycode;
         pub use decode_snappy::DecodeSnappy;
         pub use decode_zlib::DecodeZlib;
         pub use decode_zstd::DecodeZstd;
@@ -232,6 +235,7 @@ cfg_if::cfg_if! {
         pub use encode_key_value::EncodeKeyValue;
         pub use encode_logfmt::EncodeLogfmt;
         pub use encode_percent::EncodePercent;
+        pub use encode_punycode::EncodePunycode;
         pub use encode_snappy::EncodeSnappy;
         pub use encode_zlib::EncodeZlib;
         pub use encode_zstd::EncodeZstd;
@@ -388,6 +392,7 @@ pub fn all() -> Vec<Box<dyn Function>> {
         Box::new(DecodeBase64),
         Box::new(DecodeGzip),
         Box::new(DecodePercent),
+        Box::new(DecodePunycode),
         Box::new(DecodeMimeQ),
         Box::new(DecodeSnappy),
         Box::new(DecodeZlib),
@@ -402,6 +407,7 @@ pub fn all() -> Vec<Box<dyn Function>> {
         Box::new(EncodeKeyValue),
         Box::new(EncodeLogfmt),
         Box::new(EncodePercent),
+        Box::new(EncodePunycode),
         Box::new(EncodeSnappy),
         Box::new(EncodeZlib),
         Box::new(EncodeZstd),

src/stdlib/parse_url.rs

@@ -208,5 +208,35 @@ mod tests {
             })),
             tdef: TypeDef::object(inner_kind()).fallible(),
         }
+
+        punycode {
+            args: func_args![value: value!("https://www.café.com")],
+            want: Ok(value!({
+                fragment: (),
+                host: "www.xn--caf-dma.com",
+                password: "",
+                path: "/",
+                port: (),
+                query: {},
+                scheme: "https",
+                username: "",
+            })),
+            tdef: TypeDef::object(inner_kind()).fallible(),
+        }
+
+        punycode_mixed_case {
+            args: func_args![value: value!("https://www.CAFé.com")],
+            want: Ok(value!({
+                fragment: (),
+                host: "www.xn--caf-dma.com",
+                password: "",
+                path: "/",
+                port: (),
+                query: {},
+                scheme: "https",
+                username: "",
+            })),
+            tdef: TypeDef::object(inner_kind()).fallible(),
+        }
     ];
 }