Placeholder escaping (#414)

Closes #402

This adds special handling for escaped placeholders as required by the
client spec. The added tests include examples from current pages that
rely on this behavior.

The text replacements use `str::replace` which constructs a new
allocated `String`. I considered using a custom `replace_inplace` method
on `&mut str` (which works if the replacement string is at most as long
as the pattern to be replaced), but decided against it because I think
that the performance improvement is not significant enough to justify
adding `unsafe` code. It is also possible to avoid `unsafe` by
re-checking UTF-8 validity after all modifications, but the code still
felt a bit out of place for tealdeer. We can always add these
optimizations later if we want to.

src/formatter.rs

@@ -4,19 +4,52 @@ use log::debug;
 
 use crate::{extensions::FindFrom, types::LineType};
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Eq)]
 /// Represents a snippet from a page of a specific highlighting class.
-pub enum PageSnippet<'a> {
-    CommandName(&'a str),
-    Variable(&'a str),
-    NormalCode(&'a str),
-    Description(&'a str),
-    Text(&'a str),
-    Title(&'a str),
+pub enum PageSnippet<T> {
+    CommandName(T),
+    Variable(T),
+    NormalCode(T),
+    Description(T),
+    Text(T),
+    Title(T),
     Linebreak,
 }
 
-impl PageSnippet<'_> {
+#[cfg_attr(not(test), allow(dead_code))]
+impl<T> PageSnippet<T> {
+    pub fn map<F, U>(self, f: F) -> PageSnippet<U>
+    where
+        F: FnOnce(T) -> U,
+    {
+        match self {
+            PageSnippet::CommandName(s) => PageSnippet::CommandName(f(s)),
+            PageSnippet::Variable(s) => PageSnippet::Variable(f(s)),
+            PageSnippet::NormalCode(s) => PageSnippet::NormalCode(f(s)),
+            PageSnippet::Description(s) => PageSnippet::Description(f(s)),
+            PageSnippet::Text(s) => PageSnippet::Text(f(s)),
+            PageSnippet::Title(s) => PageSnippet::Title(f(s)),
+            PageSnippet::Linebreak => PageSnippet::Linebreak,
+        }
+    }
+}
+
+impl<T: PartialEq<U>, U> PartialEq<PageSnippet<U>> for PageSnippet<T> {
+    fn eq(&self, other: &PageSnippet<U>) -> bool {
+        match (self, other) {
+            (PageSnippet::CommandName(s), PageSnippet::CommandName(t))
+            | (PageSnippet::Variable(s), PageSnippet::Variable(t))
+            | (PageSnippet::NormalCode(s), PageSnippet::NormalCode(t))
+            | (PageSnippet::Description(s), PageSnippet::Description(t))
+            | (PageSnippet::Text(s), PageSnippet::Text(t))
+            | (PageSnippet::Title(s), PageSnippet::Title(t)) => s == t,
+            (PageSnippet::Linebreak, PageSnippet::Linebreak) => true,
+            _ => false,
+        }
+    }
+}
+
+impl PageSnippet<&str> {
     pub fn is_empty(&self) -> bool {
         use PageSnippet::*;
 
@@ -38,7 +71,7 @@ pub fn highlight_lines<L, F, E>(
 ) -> Result<(), E>
 where
     L: Iterator<Item = LineType>,
-    F: for<'snip> FnMut(PageSnippet<'snip>) -> Result<(), E>,
+    F: for<'snip> FnMut(PageSnippet<&'snip str>) -> Result<(), E>,
 {
     let mut command = String::new();
     for line in lines {
@@ -75,29 +108,82 @@ where
     Ok(())
 }
 
-/// Highlight code examples including user variables in {{ curly braces }}.
-fn highlight_code<'a, E>(
-    command: &'a str,
-    text: &'a str,
-    process_snippet: &mut impl FnMut(PageSnippet<'a>) -> Result<(), E>,
+/// Highlight code examples.
+/// - parse placeholders (`{{ curly braces }}`)
+/// - replace escaped placeholder markers (`\{\{` and `\}\}`)
+fn highlight_code<E>(
+    command: &str,
+    mut text: &str,
+    process_snippet: &mut impl FnMut(PageSnippet<&str>) -> Result<(), E>,
 ) -> Result<(), E> {
-    let variable_splits = text
-        .split("}}")
-        .map(|s| s.split_once("{{").unwrap_or((s, "")));
-    for (code_segment, variable) in variable_splits {
-        highlight_code_segment(command, code_segment, process_snippet)?;
-        process_snippet(PageSnippet::Variable(variable))?;
+    // We replace escaped placeholder markers at the end so that our replacing does not interfere
+    // with finding the actual markers.
+    // NOTE: This is not optimal, as it allocates one String for each `replace`
+    let replace_escaped = |s: &str| s.replace(r"\{\{", "{{").replace(r"\}\}", "}}");
+
+    loop {
+        // Find placeholder markers and split into code and placeholder accordingly
+
+        let Some(start_marker) = find_marker(text, "{{", r"\{\{") else {
+            break;
+        };
+        let Some(mut end_marker) = find_marker(&text[start_marker + 2..], "}}", r"\}\}") else {
+            break;
+        };
+        end_marker += start_marker + 2;
+
+        // Greedily extend matched range
+        while end_marker + 2 < text.len() && text.as_bytes()[end_marker + 2] == b'}' {
+            end_marker += 1;
+        }
+
+        let placeholder_content = &text[start_marker + 2..end_marker];
+
+        if start_marker > 0 {
+            highlight_code_segment(
+                command,
+                &replace_escaped(&text[..start_marker]),
+                process_snippet,
+            )?;
+        }
+        process_snippet(PageSnippet::Variable(&replace_escaped(placeholder_content)))?;
+
+        text = &text[end_marker + 2..];
     }
+
+    if !text.is_empty() {
+        highlight_code_segment(command, &replace_escaped(text), process_snippet)?;
+    }
+
     Ok(())
 }
 
+/// Find a "{{" (or "}}") substring that does not overlap with a preceding "\{\{" (or "\}\}").
+fn find_marker(s: &str, marker: &str, forbidden_prefix: &str) -> Option<usize> {
+    let mut search_start = 0;
+    loop {
+        let marker_index = s.find_from(marker, search_start)?;
+
+        let overlaps_with_prefix = (forbidden_prefix.len() <= marker_index + 1) && {
+            let prefix_start = marker_index + 1 - forbidden_prefix.len();
+            &s[prefix_start..=marker_index] == forbidden_prefix
+        };
+        if !overlaps_with_prefix {
+            return Some(marker_index);
+        }
+
+        // The next valid marker cannot include the first character of the current match
+        search_start = marker_index + 1;
+    }
+}
+
 /// Yields `NormalCode` and `CommandName` in alternating order according to the occurrences of
 /// `command_name` in `segment`. Variables are not detected here, see `highlight_code`
 /// instead.
 fn highlight_code_segment<'a, E>(
     command_name: &'a str,
     mut segment: &'a str,
-    process_snippet: &mut impl FnMut(PageSnippet<'a>) -> Result<(), E>,
+    process_snippet: &mut impl FnMut(PageSnippet<&'a str>) -> Result<(), E>,
 ) -> Result<(), E> {
     if !command_name.is_empty() {
         let mut search_start = 0;
@@ -140,7 +226,6 @@ fn is_freestanding_substring(surrounding: &str, substring: (usize, usize)) -> bo
 #[cfg(test)]
 mod tests {
     use super::*;
-    use PageSnippet::*;
 
     #[test]
     fn test_is_freestanding_substring() {
@@ -167,80 +252,186 @@ mod tests {
         ));
     }
 
-    fn run<'a>(cmd: &'a str, segment: &'a str) -> Vec<PageSnippet<'a>> {
+    fn run<'a>(cmd: &'a str, segment: &'a str) -> Vec<PageSnippet<String>> {
         let mut yielded = Vec::new();
-        let mut process_snippet = |snip: PageSnippet<'a>| {
+        let mut process_snippet = |snip: PageSnippet<&str>| {
             if !snip.is_empty() {
-                yielded.push(snip);
+                yielded.push(snip.map(str::to_string));
             }
             Ok::<(), ()>(())
         };
 
-        highlight_code_segment(cmd, segment, &mut process_snippet)
-            .expect("highlight code segment failed");
+        highlight_code(cmd, segment, &mut process_snippet).expect("highlight code segment failed");
         yielded
     }
 
-    #[test]
-    fn test_highlight_code_segment() {
-        assert!(run("make", "").is_empty());
-        assert_eq!(
-            &run("make", "make all CC=clang -q"),
-            &[CommandName("make"), NormalCode(" all CC=clang -q")]
-        );
-        assert_eq!(
-            &run("make", "  make money --always-make"),
-            &[
-                NormalCode("  "),
-                CommandName("make"),
-                NormalCode(" money --always-make")
-            ]
-        );
-        assert_eq!(
-            &run("git commit", "git commit -m 'git commit'"),
-            &[CommandName("git commit"), NormalCode(" -m 'git commit'"),]
-        );
+    mod highlight_code_segment {
+        use super::*;
+        use PageSnippet::*;
+
+        #[test]
+        fn test_highlight_code_segment() {
+            assert!(run("make", "").is_empty());
+            assert_eq!(
+                &run("make", "make all CC=clang -q"),
+                &[CommandName("make"), NormalCode(" all CC=clang -q")]
+            );
+            assert_eq!(
+                &run("make", "  make money --always-make"),
+                &[
+                    NormalCode("  "),
+                    CommandName("make"),
+                    NormalCode(" money --always-make")
+                ]
+            );
+            assert_eq!(
+                &run("git commit", "git commit -m 'git commit'"),
+                &[CommandName("git commit"), NormalCode(" -m 'git commit'"),]
+            );
+        }
+
+        #[test]
+        fn test_i18n() {
+            assert_eq!(
+                &run("mäke", "mäke höhlenrätselbücher"),
+                &[CommandName("mäke"), NormalCode(" höhlenrätselbücher")]
+            );
+            assert_eq!(
+                &run(
+                    "Müll",
+                    "1000 Gründe warum Müll heute größer ist als Müll früher, ärgerlich"
+                ),
+                &[
+                    NormalCode("1000 Gründe warum "),
+                    CommandName("Müll"),
+                    NormalCode(" heute größer ist als "),
+                    CommandName("Müll"),
+                    NormalCode(" früher, ärgerlich")
+                ]
+            );
+            assert_eq!(
+                &run(
+                    "übergang",
+                    "die Zustandsübergangsfunktion übergang Änderungen",
+                ),
+                &[
+                    NormalCode("die Zustandsübergangsfunktion "),
+                    CommandName("übergang"),
+                    NormalCode(" Änderungen")
+                ],
+            );
+        }
+
+        #[test]
+        fn test_empty_command() {
+            let segment = "some code";
+            let snippets = [NormalCode(segment)];
+
+            assert_eq!(run("", segment), snippets);
+            assert_eq!(run(" ", segment), snippets);
+            assert_eq!(run("  \t ", segment), snippets);
+        }
     }
 
-    #[test]
-    fn test_i18n() {
-        assert_eq!(
-            &run("mäke", "mäke höhlenrätselbücher"),
-            &[CommandName("mäke"), NormalCode(" höhlenrätselbücher")]
-        );
-        assert_eq!(
-            &run(
-                "Müll",
-                "1000 Gründe warum Müll heute größer ist als Müll früher, ärgerlich"
-            ),
-            &[
-                NormalCode("1000 Gründe warum "),
-                CommandName("Müll"),
-                NormalCode(" heute größer ist als "),
-                CommandName("Müll"),
-                NormalCode(" früher, ärgerlich")
-            ]
-        );
-        assert_eq!(
-            &run(
-                "übergang",
-                "die Zustandsübergangsfunktion übergang Änderungen",
-            ),
-            &[
-                NormalCode("die Zustandsübergangsfunktion "),
-                CommandName("übergang"),
-                NormalCode(" Änderungen")
-            ],
-        );
-    }
+    mod placeholders {
+        use super::*;
+        use PageSnippet::*;
 
-    #[test]
-    fn test_empty_command() {
-        let segment = "some code";
-        let snippets = [NormalCode(segment)];
+        #[test]
+        fn variable_vs_escaped() {
+            assert_eq!(
+                run("ping", "ping {{example.com}}"),
+                [
+                    CommandName("ping"),
+                    NormalCode(" "),
+                    Variable("example.com"),
+                ],
+            );
+            assert_eq!(
+                run(
+                    "docker inspect",
+                    r"docker inspect --format '\{\{range.NetworkSettings.Networks\}\}\{\{.IPAddress\}\}\{\{end\}\}' {{container}}"
+                ),
+                [
+                    CommandName("docker inspect"),
+                    NormalCode(
+                        " --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "
+                    ),
+                    Variable("container"),
+                ],
+            );
+            assert_eq!(
+                run("mount", r"mount \\{{computer_name}}\{{share_name}} Z:"),
+                [
+                    CommandName("mount"),
+                    NormalCode(r" \\"),
+                    Variable("computer_name"),
+                    NormalCode(r"\"),
+                    Variable("share_name"),
+                    NormalCode(" Z:"),
+                ],
+            );
 
-        assert_eq!(run("", segment), snippets);
-        assert_eq!(run(" ", segment), snippets);
-        assert_eq!(run("  \t ", segment), snippets);
+            assert_eq!(run("", r"\{"), [NormalCode(r"\{")]);
+            assert_eq!(run("", r"\{{a"), [NormalCode(r"\{{a")]);
+            assert_eq!(run("", r"\{{a}}"), [NormalCode(r"\"), Variable("a")]);
+
+            // Placeholder has begin marker, but no end marker
+            assert_eq!(run("", r"{{\}\}}"), [NormalCode("{{}}}")]);
+        }
+
+        #[test]
+        fn outer_precedence() {
+            assert_eq!(
+                run("git stash", "git stash show --patch {{stash@{0}}}"),
+                [
+                    CommandName("git stash"),
+                    NormalCode(" show --patch "),
+                    Variable("stash@{0}"),
+                ],
+            );
+
+            // The following is not listed in the specification, but this is the highlighting I would expect.
+            assert_eq!(
+                run("rg", "rg {{}}}"),
+                [CommandName("rg"), NormalCode(" "), Variable("}")]
+            );
+
+            // And these are just to document the current behavior
+            assert_eq!(run("", "{{{}}}"), [Variable("{}")]);
+            assert_eq!(run("", "{{{{}}}"), [Variable("{{}")]);
+            assert_eq!(run("", "{{{}}}}"), [Variable("{}}")]);
+        }
+
+        #[test]
+        fn escaped_inside_placeholder() {
+            assert_eq!(
+                run(
+                    "playerctl",
+                    r#"playerctl metadata {{[-f|--format]}} "{{Now playing: \{\{artist\}\} - \{\{album\}\} - \{\{title\}\}}}""#
+                ),
+                [
+                    CommandName("playerctl"),
+                    NormalCode(" metadata "),
+                    Variable("[-f|--format]"),
+                    NormalCode(" \""),
+                    Variable("Now playing: {{artist}} - {{album}} - {{title}}"),
+                    NormalCode("\""),
+                ],
+            );
+        }
+
+        #[test]
+        fn placeholder_inside_escaped() {
+            assert_eq!(
+                run("test", r#"test \{\{{{var}} normal\}\}"#),
+                [
+                    CommandName("test"),
+                    NormalCode(" {{"),
+                    Variable("var"),
+                    NormalCode(" normal}}"),
+                ],
+            );
+        }
     }
 }

src/output.rs

@@ -56,7 +56,7 @@ pub fn print_page(
         }
     } else {
         // Closure that processes a page snippet and writes it to stdout
-        let mut process_snippet = |snip: PageSnippet<'_>| {
+        let mut process_snippet = |snip: PageSnippet<&str>| {
             if snip.is_empty() {
                 Ok(())
             } else {
@@ -82,7 +82,7 @@ pub fn print_page(
 
 fn print_snippet(
     writer: &mut impl Write,
-    snip: PageSnippet<'_>,
+    snip: PageSnippet<&str>,
     style: &StyleConfig,
 ) -> io::Result<()> {
     use PageSnippet::*;