From cdbca5c53b2d2f0c3fd1189a2def4cff3c58ea4d Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Fri, 21 May 2021 12:36:32 +0200
Subject: [PATCH 1/8] Add `FindFrom` extension trait to `str` to start
 searching at a byte offset.

This also moves the contents of `dedup.rs` into a shared module `extensions`.
---
 src/{dedup.rs => extensions.rs} | 13 +++++++++++++
 src/main.rs                     |  4 ++--
 2 files changed, 15 insertions(+), 2 deletions(-)
 rename src/{dedup.rs => extensions.rs} (61%)

diff --git a/src/dedup.rs b/src/extensions.rs
similarity index 61%
rename from src/dedup.rs
rename to src/extensions.rs
index b06fe0c..3aebfa2 100644
--- a/src/dedup.rs
+++ b/src/extensions.rs
@@ -18,3 +18,16 @@ impl<T: PartialEq + Clone> Dedup<T> for Vec<T> {
         }
     }
 }
+
+/// Like `str::find`, but starts searching at `start`.
+pub(crate) trait FindFrom {
+    fn find_from(&self, needle: &Self, start: usize) -> Option<usize>;
+}
+
+impl FindFrom for str {
+    fn find_from(&self, needle: &Self, start: usize) -> Option<usize> {
+        self.get(start..)
+            .and_then(|s| s.find(needle))
+            .map(|i| i + start)
+    }
+}
diff --git a/src/main.rs b/src/main.rs
index 693b793..0d68b06 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -31,16 +31,16 @@ use serde_derive::Deserialize;
 
 mod cache;
 mod config;
-mod dedup;
 mod error;
+pub mod extensions;
 mod formatter;
 mod tokenizer;
 mod types;
 
 use crate::cache::{Cache, PageLookupResult};
 use crate::config::{get_config_dir, get_config_path, make_default_config, Config, MAX_CACHE_AGE};
-use crate::dedup::Dedup;
 use crate::error::TealdeerError::ConfigError;
+use crate::extensions::Dedup;
 use crate::formatter::print_lines;
 use crate::tokenizer::Tokenizer;
 use crate::types::{ColorOptions, OsType};

From 62e82461cb182f163d2f5e4d87186b34ed1c5713 Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Fri, 21 May 2021 12:38:44 +0200
Subject: [PATCH 2/8] Refactor most of `formatter.rs`

This includes
- adding the `HighlightingSnippet` enum as a common ground for
  highlighting and printing code to communicate
- decomposing `print_lines` accordingly
- clearing up `highlight_code_segment` (previously `highlight_command`)
- adding unit tests (now that they can reason about
  `HighlightingSnippet`s instead of having to output on the integration
  test level
---
 src/formatter.rs | 274 ++++++++++++++++++++++++++++++++++++-----------
 src/main.rs      |  12 ++-
 2 files changed, 221 insertions(+), 65 deletions(-)

diff --git a/src/formatter.rs b/src/formatter.rs
index 9859c1a..dd7335d 100644
--- a/src/formatter.rs
+++ b/src/formatter.rs
@@ -1,84 +1,122 @@
 //! Functions related to formatting and printing lines from a `Tokenizer`.
 
-use std::io::{BufRead, Write};
+use std::io::{self, BufRead, Write};
 
-use ansi_term::{ANSIString, ANSIStrings};
 use log::debug;
 
-use crate::config::Config;
+use crate::config::StyleConfig;
 use crate::error::TealdeerError::{self, WriteError};
+use crate::extensions::FindFrom;
 use crate::tokenizer::Tokenizer;
 use crate::types::LineType;
 
-fn highlight_command<'a>(
-    command: &'a str,
-    example_code: &'a str,
-    config: &Config,
-    parts: &mut Vec<ANSIString<'a>>,
-) {
-    let mut code_part_end_pos = 0;
-    while let Some(command_start) = example_code[code_part_end_pos..].find(&command) {
-        let code_part = &example_code[code_part_end_pos..code_part_end_pos + command_start];
-        parts.push(config.style.example_code.paint(code_part));
-        if code_part_end_pos == 0 {
-            // Only highlight command names at the start of the line ...
-            parts.push(config.style.command_name.paint(command));
-        } else {
-            let char_before_command = example_code
-                .chars()
-                .nth(code_part_end_pos + command_start - 1);
-            if char_before_command.filter(|c| c.is_whitespace()).is_some() {
-                // ... or when preceded by a whitespace character.
-                parts.push(config.style.command_name.paint(command));
-            } else {
-                parts.push(config.style.example_code.paint(command));
-            }
-        }
-
-        code_part_end_pos += command_start + command.len();
-    }
-    parts.push(
-        config
-            .style
-            .example_code
-            .paint(&example_code[code_part_end_pos..]),
-    );
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum HighlightingSnippet<'a> {
+    CommandName(&'a str),
+    Variable(&'a str),
+    NormalCode(&'a str),
+    Description(&'a str),
+    Text(&'a str),
+    Linebreak,
 }
 
-/// Format and highlight code examples including variables in {{ curly braces }}.
-fn format_code(command: &str, text: &str, config: &Config) -> String {
-    let mut parts = Vec::new();
-    for between_variables in text.split("}}") {
-        if let Some(variable_start) = between_variables.find("{{") {
-            let example_code = &between_variables[..variable_start];
-            let example_variable = &between_variables[variable_start + 2..];
+impl<'a> HighlightingSnippet<'a> {
+    pub fn is_empty(&self) -> bool {
+        use HighlightingSnippet::*;
 
-            highlight_command(command, example_code, config, &mut parts);
-            parts.push(config.style.example_variable.paint(example_variable));
-        } else {
-            highlight_command(command, between_variables, config, &mut parts);
+        match self {
+            CommandName(s) | Variable(s) | NormalCode(s) | Description(s) | Text(s) => s.is_empty(),
+            Linebreak => false,
         }
     }
+}
 
-    ANSIStrings(&parts).to_string()
+/// Checks whether the characters right before and after the substring (given by half-open index interval) are whitespace (if they exist).
+fn is_freestanding_substring(surrouding: &str, substring: (usize, usize)) -> bool {
+    let (start, end) = substring;
+    // "okay" meaning <exists and is whitespace> or <doesn't exist>
+    let char_before_is_okay = surrouding[..start]
+        .chars()
+        .last()
+        .filter(|prev_char| !prev_char.is_whitespace())
+        .is_none();
+    let char_after_is_okay = surrouding[end..]
+        .chars()
+        .next()
+        .filter(|next_char| !next_char.is_whitespace())
+        .is_none();
+    char_before_is_okay && char_after_is_okay
+}
+
+/// Yields `NormalCode` and `CommandName` in alternating order according to the occurences of
+/// `command_name` in `segment`. Variables are not detected here, see `highlight_code`
+/// instead.
+fn highlight_code_segment<'a, E>(
+    command_name: &'a str,
+    mut segment: &'a str,
+    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
+) -> Result<(), E> {
+    if !command_name.is_empty() {
+        let mut search_start = 0;
+        while let Some(match_start) = segment.find_from(command_name, search_start) {
+            let match_end = match_start + command_name.len();
+            if is_freestanding_substring(segment, (match_start, match_end)) {
+                yield_snippet(HighlightingSnippet::NormalCode(&segment[..match_start]))?;
+                yield_snippet(HighlightingSnippet::CommandName(command_name))?;
+                segment = &segment[match_end..];
+                search_start = 0;
+            } else {
+                search_start = segment[match_start..]
+                    .char_indices()
+                    .nth(1)
+                    .map_or(segment.len(), |(i, _)| match_start + i);
+            }
+        }
+    }
+    yield_snippet(HighlightingSnippet::NormalCode(segment))?;
+    Ok(())
+}
+
+/// Highlight code examples including user variables in {{ curly braces }}.
+fn highlight_code<'a, E>(
+    command: &'a str,
+    text: &'a str,
+    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
+) -> Result<(), E> {
+    let variable_splits = text
+        .split("}}")
+        .map(|s| s.split_once("{{").unwrap_or((s, "")));
+    for (code_segment, variable) in variable_splits {
+        highlight_code_segment(&command, code_segment, yield_snippet)?;
+        yield_snippet(HighlightingSnippet::Variable(variable))?;
+    }
+    Ok(())
 }
 
 /// Print a token stream to an ANSI terminal.
 pub fn print_lines<T, R>(
     writer: &mut T,
     tokenizer: &mut Tokenizer<R>,
-    config: &Config,
+    style: &StyleConfig,
+    keep_empty_lines: bool,
 ) -> Result<(), TealdeerError>
 where
     T: Write,
     R: BufRead,
 {
     let mut command = String::new();
+    let mut yield_snippet = |snip: HighlightingSnippet<'_>| {
+        if snip.is_empty() {
+            Ok(())
+        } else {
+            print_snippet(writer, snip, style).map_err(|e| WriteError(e.to_string()))
+        }
+    };
     while let Some(token) = tokenizer.next_token() {
         match token {
             LineType::Empty => {
-                if !config.display.compact {
-                    writeln!(writer).map_err(|e| WriteError(e.to_string()))?;
+                if keep_empty_lines {
+                    yield_snippet(HighlightingSnippet::Linebreak)?;
                 }
             }
             LineType::Title(title) => {
@@ -89,20 +127,132 @@ where
                 command = title;
                 debug!("Detected command name: {}", &command);
             }
-            LineType::Description(text) => {
-                writeln!(writer, "  {}", config.style.description.paint(text))
-                    .map_err(|e| WriteError(e.to_string()))?;
-            }
-            LineType::ExampleText(text) => {
-                writeln!(writer, "  {}", config.style.example_text.paint(text))
-                    .map_err(|e| WriteError(e.to_string()))?;
-            }
+            LineType::Description(text) => yield_snippet(HighlightingSnippet::Description(&text))?,
+            LineType::ExampleText(text) => yield_snippet(HighlightingSnippet::Text(&text))?,
             LineType::ExampleCode(text) => {
-                writeln!(writer, "      {}", format_code(&command, &text, config))
-                    .map_err(|e| WriteError(e.to_string()))?;
+                yield_snippet(HighlightingSnippet::NormalCode("      "))?;
+                highlight_code(&command, &text, &mut yield_snippet)?;
+                yield_snippet(HighlightingSnippet::Linebreak)?;
             }
+
             LineType::Other(text) => debug!("Unknown line type: {:?}", text),
         }
     }
-    writeln!(writer).map_err(|e| WriteError(e.to_string()))
+    yield_snippet(HighlightingSnippet::Linebreak)?;
+    Ok(())
+}
+
+fn print_snippet(
+    writer: &mut impl Write,
+    snip: HighlightingSnippet<'_>,
+    style: &StyleConfig,
+) -> Result<(), io::Error> {
+    use HighlightingSnippet::*;
+
+    match snip {
+        CommandName(s) => write!(writer, "{}", style.command_name.paint(s)),
+        Variable(s) => write!(writer, "{}", style.example_variable.paint(s)),
+        NormalCode(s) => write!(writer, "{}", style.example_code.paint(s)),
+        Description(s) => writeln!(writer, "  {}", style.description.paint(s)),
+        Text(s) => writeln!(writer, "  {}", style.example_text.paint(s)),
+        Linebreak => writeln!(writer),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use HighlightingSnippet::*;
+
+    #[test]
+    fn test_is_freestanding_substring() {
+        assert!(is_freestanding_substring("I love tldr", (0, 1)));
+        assert!(is_freestanding_substring("I love tldr", (2, 6)));
+        assert!(is_freestanding_substring("I love tldr", (7, 11)));
+
+        assert!(is_freestanding_substring("tldr", (0, 4)));
+        assert!(is_freestanding_substring("tldr ", (0, 4)));
+        assert!(is_freestanding_substring(" tldr", (1, 5)));
+        assert!(is_freestanding_substring(" tldr ", (1, 5)));
+
+        assert!(!is_freestanding_substring("tldr", (1, 3)));
+        assert!(!is_freestanding_substring("tldr ", (1, 4)));
+        assert!(!is_freestanding_substring(" tldr", (1, 4)));
+
+        assert!(is_freestanding_substring(
+            " épicé ",
+            (1, " épicé".len()) // note the missing trailing space
+        ));
+        assert!(!is_freestanding_substring(
+            " épicé ",
+            (1, " épic".len()) // note the missing trailing space and character
+        ));
+    }
+
+    fn run<'a>(cmd: &'a str, segment: &'a str) -> Vec<HighlightingSnippet<'a>> {
+        let mut yielded = Vec::new();
+        let mut yield_snippet = |snip: HighlightingSnippet<'a>| {
+            if !snip.is_empty() {
+                yielded.push(snip);
+            }
+            Ok::<(), ()>(())
+        };
+
+        highlight_code_segment(cmd, segment, &mut yield_snippet)
+            .expect("highlight code segment failed");
+        yielded
+    }
+
+    #[test]
+    fn test_highlight_code_segment() {
+        assert!(run("make", "").is_empty());
+        assert_eq!(
+            &run("make", "make all CC=clang -q"),
+            &[CommandName("make"), NormalCode(" all CC=clang -q")]
+        );
+        assert_eq!(
+            &run("make", "  make money --always-make"),
+            &[
+                NormalCode("  "),
+                CommandName("make"),
+                NormalCode(" money --always-make")
+            ]
+        );
+        assert_eq!(
+            &run("git commit", "git commit -m 'git commit'"),
+            &[CommandName("git commit"), NormalCode(" -m 'git commit'"),]
+        );
+    }
+
+    #[test]
+    fn test_i18n() {
+        assert_eq!(
+            &run("mäke", "mäke höhlenrätselbücher"),
+            &[CommandName("mäke"), NormalCode(" höhlenrätselbücher")]
+        );
+        assert_eq!(
+            &run(
+                "Müll",
+                "1000 Gründe warum Müll heute größer ist als Müll früher, ärgerlich"
+            ),
+            &[
+                NormalCode("1000 Gründe warum "),
+                CommandName("Müll"),
+                NormalCode(" heute größer ist als "),
+                CommandName("Müll"),
+                NormalCode(" früher, ärgerlich")
+            ]
+        );
+        assert_eq!(
+            &run(
+                "übergang",
+                "die Zustandsübergangsfunktion übergang Änderungen",
+            ),
+            &[
+                NormalCode("die Zustandsübergangsfunktion "),
+                CommandName("übergang"),
+                NormalCode(" Änderungen")
+            ],
+        );
+    }
 }
diff --git a/src/main.rs b/src/main.rs
index 0d68b06..ffdfb92 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -10,8 +10,9 @@
 
 #![deny(clippy::all)]
 #![warn(clippy::pedantic)]
-#![allow(clippy::similar_names)]
+#![allow(clippy::enum_glob_use)]
 #![allow(clippy::module_name_repetitions)]
+#![allow(clippy::similar_names)]
 #![allow(clippy::too_many_lines)]
 
 use std::fs::File;
@@ -99,8 +100,13 @@ fn print_page(
         } else {
             // Create tokenizer and print output
             let mut tokenizer = Tokenizer::new(reader);
-            print_lines(&mut handle, &mut tokenizer, config)
-                .map_err(|e| format!("Could not write to stdout: {}", e.message()))?;
+            print_lines(
+                &mut handle,
+                &mut tokenizer,
+                &config.style,
+                !config.display.compact,
+            )
+            .map_err(|e| format!("Could not write to stdout: {}", e.message()))?;
         };
     }
 

From e8b1c9e801cc07348e2d7e550fd5a6b473479bbd Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Wed, 2 Jun 2021 21:04:05 +0200
Subject: [PATCH 3/8] Rename `Tokenizer` to `LineIterator`

This also replaces the `next_token` method with the `next` method from
the `Iterator` trait.
---
 src/formatter.rs                       | 14 +++---
 src/{tokenizer.rs => line_iterator.rs} | 68 +++++++++++++-------------
 src/main.rs                            |  8 ++-
 3 files changed, 44 insertions(+), 46 deletions(-)
 rename src/{tokenizer.rs => line_iterator.rs} (53%)

diff --git a/src/formatter.rs b/src/formatter.rs
index dd7335d..9a26c1d 100644
--- a/src/formatter.rs
+++ b/src/formatter.rs
@@ -1,13 +1,12 @@
 //! Functions related to formatting and printing lines from a `Tokenizer`.
 
-use std::io::{self, BufRead, Write};
+use std::io::{self, Write};
 
 use log::debug;
 
 use crate::config::StyleConfig;
 use crate::error::TealdeerError::{self, WriteError};
 use crate::extensions::FindFrom;
-use crate::tokenizer::Tokenizer;
 use crate::types::LineType;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -94,15 +93,14 @@ fn highlight_code<'a, E>(
 }
 
 /// Print a token stream to an ANSI terminal.
-pub fn print_lines<T, R>(
+pub fn print_lines<T>(
     writer: &mut T,
-    tokenizer: &mut Tokenizer<R>,
+    lines: impl Iterator<Item = LineType>,
     style: &StyleConfig,
     keep_empty_lines: bool,
 ) -> Result<(), TealdeerError>
 where
     T: Write,
-    R: BufRead,
 {
     let mut command = String::new();
     let mut yield_snippet = |snip: HighlightingSnippet<'_>| {
@@ -112,8 +110,8 @@ where
             print_snippet(writer, snip, style).map_err(|e| WriteError(e.to_string()))
         }
     };
-    while let Some(token) = tokenizer.next_token() {
-        match token {
+    for line in lines {
+        match line {
             LineType::Empty => {
                 if keep_empty_lines {
                     yield_snippet(HighlightingSnippet::Linebreak)?;
@@ -123,7 +121,7 @@ where
                 debug!("Ignoring title");
 
                 // This is safe as long as the parsed title is only the command,
-                // and tokenizer yields values in order of appearance.
+                // and the iterator yields values in order of appearance.
                 command = title;
                 debug!("Detected command name: {}", &command);
             }
diff --git a/src/tokenizer.rs b/src/line_iterator.rs
similarity index 53%
rename from src/tokenizer.rs
rename to src/line_iterator.rs
index f9b3837..c1fbb0f 100644
--- a/src/tokenizer.rs
+++ b/src/line_iterator.rs
@@ -1,4 +1,4 @@
-//! Code to tokenize a `BufRead` instance into an iterator of `LineType`s.
+//! Code to split a `BufRead` instance into an iterator of `LineType`s.
 
 use std::io::BufRead;
 
@@ -15,13 +15,13 @@ pub enum TldrFormat {
     V2,
 }
 
-/// A tokenizer is initialized with a `BufReader` instance that contains the
-/// entire Tldr page. It then returns tokens as `Option<LineType>`.
+/// A `LineIterator` is initialized with a `BufReader` instance that contains the
+/// entire Tldr page. It then implements `Iterator<Item = LineType>`.
 #[derive(Debug)]
-pub struct Tokenizer<R: BufRead> {
+pub struct LineIterator<R: BufRead> {
     /// An instance of `R: BufRead`.
     reader: R,
-    /// Whether the first line has already been tokenized or not.
+    /// Whether the first line has already been processed or not.
     first_line: bool,
     /// Buffer for the current line. Used internally.
     current_line: String,
@@ -29,7 +29,7 @@ pub struct Tokenizer<R: BufRead> {
     format: TldrFormat,
 }
 
-impl<R> Tokenizer<R>
+impl<R> LineIterator<R>
 where
     R: BufRead,
 {
@@ -41,38 +41,40 @@ where
             format: TldrFormat::Undecided,
         }
     }
+}
 
-    pub fn next_token(&mut self) -> Option<LineType> {
+impl<R: BufRead> Iterator for LineIterator<R> {
+    type Item = LineType;
+
+    fn next(&mut self) -> Option<LineType> {
         self.current_line.clear();
         let bytes_read = self.reader.read_line(&mut self.current_line);
         match bytes_read {
             Ok(0) => None,
             Err(e) => {
-                warn!("Could not read line from token reader: {:?}", e);
+                warn!("Could not read line from reader: {:?}", e);
                 None
             }
             Ok(_) => {
                 // Handle new titles
-                if self.first_line && !self.current_line.starts_with('#') {
-                    // It's the new format! Drop next line.
-                    // (Hmm, is there a way to do this without an allocation?)
-                    let mut devnull = String::new();
-                    if let Err(e) = self.reader.read_line(&mut devnull) {
-                        warn!("Could not read line from token reader: {:?}", e);
-                        return None;
-                    }
-                    self.first_line = false;
-                    self.format = TldrFormat::V2;
-                    return Some(LineType::Title(self.current_line.trim_end().to_string()));
-                }
-
                 if self.first_line {
-                    // Clear `first_line` flag
-                    self.first_line = false;
-
-                    // It's the old format.
-                    self.format = TldrFormat::V1;
+                    if self.current_line.starts_with('#') {
+                        // It's the old format.
+                        self.format = TldrFormat::V1;
+                    } else {
+                        // It's the new format! Drop next line.
+                        // (Hmm, is there a way to do this without an allocation?)
+                        let mut devnull = String::new();
+                        if let Err(e) = self.reader.read_line(&mut devnull) {
+                            warn!("Could not read line from reader: {:?}", e);
+                            return None;
+                        }
+                        self.first_line = false;
+                        self.format = TldrFormat::V2;
+                        return Some(LineType::Title(self.current_line.trim_end().to_string()));
+                    }
                 }
+                self.first_line = false;
 
                 // Convert line to a `LineType` instance
                 match self.format {
@@ -87,26 +89,26 @@ where
 
 #[cfg(test)]
 mod test {
-    use super::Tokenizer;
+    use super::LineIterator;
     use crate::types::LineType;
 
     #[test]
     fn test_first_line_old_format() {
         let input = "# The Title\n\n";
-        let mut tokenizer = Tokenizer::new(input.as_bytes());
-        let title = tokenizer.next_token().unwrap();
+        let mut lines = LineIterator::new(input.as_bytes());
+        let title = lines.next().unwrap();
         assert_eq!(title, LineType::Title("The Title".to_string()));
-        let empty = tokenizer.next_token().unwrap();
+        let empty = lines.next().unwrap();
         assert_eq!(empty, LineType::Empty);
     }
 
     #[test]
     fn test_first_line_new_format() {
         let input = "The Title\n=========\n\n";
-        let mut tokenizer = Tokenizer::new(input.as_bytes());
-        let title = tokenizer.next_token().unwrap();
+        let mut lines = LineIterator::new(input.as_bytes());
+        let title = lines.next().unwrap();
         assert_eq!(title, LineType::Title("The Title".to_string()));
-        let empty = tokenizer.next_token().unwrap();
+        let empty = lines.next().unwrap();
         assert_eq!(empty, LineType::Empty);
     }
 }
diff --git a/src/main.rs b/src/main.rs
index ffdfb92..be65e26 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -35,7 +35,7 @@ mod config;
 mod error;
 pub mod extensions;
 mod formatter;
-mod tokenizer;
+mod line_iterator;
 mod types;
 
 use crate::cache::{Cache, PageLookupResult};
@@ -43,7 +43,7 @@ use crate::config::{get_config_dir, get_config_path, make_default_config, Config
 use crate::error::TealdeerError::ConfigError;
 use crate::extensions::Dedup;
 use crate::formatter::print_lines;
-use crate::tokenizer::Tokenizer;
+use crate::line_iterator::LineIterator;
 use crate::types::{ColorOptions, OsType};
 
 const NAME: &str = "tealdeer";
@@ -98,11 +98,9 @@ fn print_page(
                     .map_err(|_| "Could not write to stdout".to_string())?;
             }
         } else {
-            // Create tokenizer and print output
-            let mut tokenizer = Tokenizer::new(reader);
             print_lines(
                 &mut handle,
-                &mut tokenizer,
+                LineIterator::new(reader),
                 &config.style,
                 !config.display.compact,
             )

From 808ad7ff303ef21c43de08eb350ce1cf4b6f12d7 Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Wed, 2 Jun 2021 22:01:02 +0200
Subject: [PATCH 4/8] Extract output logic for pages into own module

This moves `print_page` from `main.rs` and `print_snippet` from
`formatter.rs` into a new file `output.rs`. To decompose `print_snippet`
from `print_lines`, the latter now takes the `yield_snippet` callable as
an argument (similar to how the helper methods got a hold of it). In
order for this to work, callers have to provide a function that is
generic over all possible snippet lifetimes.
---
 src/formatter.rs | 48 +++++++-------------------------
 src/main.rs      | 46 ++-----------------------------
 src/output.rs    | 71 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+), 81 deletions(-)
 create mode 100644 src/output.rs

diff --git a/src/formatter.rs b/src/formatter.rs
index 9a26c1d..54d805d 100644
--- a/src/formatter.rs
+++ b/src/formatter.rs
@@ -1,16 +1,12 @@
 //! Functions related to formatting and printing lines from a `Tokenizer`.
 
-use std::io::{self, Write};
-
-use log::debug;
-
-use crate::config::StyleConfig;
-use crate::error::TealdeerError::{self, WriteError};
 use crate::extensions::FindFrom;
 use crate::types::LineType;
 
+use log::debug;
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum HighlightingSnippet<'a> {
+pub enum HighlightingSnippet<'a> {
     CommandName(&'a str),
     Variable(&'a str),
     NormalCode(&'a str),
@@ -93,23 +89,16 @@ fn highlight_code<'a, E>(
 }
 
 /// Print a token stream to an ANSI terminal.
-pub fn print_lines<T>(
-    writer: &mut T,
-    lines: impl Iterator<Item = LineType>,
-    style: &StyleConfig,
+pub fn highlight_lines<L, F, E>(
+    lines: L,
+    yield_snippet: &mut F,
     keep_empty_lines: bool,
-) -> Result<(), TealdeerError>
+) -> Result<(), E>
 where
-    T: Write,
+    L: Iterator<Item = LineType>,
+    F: for<'snip> FnMut(HighlightingSnippet<'snip>) -> Result<(), E>,
 {
     let mut command = String::new();
-    let mut yield_snippet = |snip: HighlightingSnippet<'_>| {
-        if snip.is_empty() {
-            Ok(())
-        } else {
-            print_snippet(writer, snip, style).map_err(|e| WriteError(e.to_string()))
-        }
-    };
     for line in lines {
         match line {
             LineType::Empty => {
@@ -129,7 +118,7 @@ where
             LineType::ExampleText(text) => yield_snippet(HighlightingSnippet::Text(&text))?,
             LineType::ExampleCode(text) => {
                 yield_snippet(HighlightingSnippet::NormalCode("      "))?;
-                highlight_code(&command, &text, &mut yield_snippet)?;
+                highlight_code(&command, &text, yield_snippet)?;
                 yield_snippet(HighlightingSnippet::Linebreak)?;
             }
 
@@ -140,23 +129,6 @@ where
     Ok(())
 }
 
-fn print_snippet(
-    writer: &mut impl Write,
-    snip: HighlightingSnippet<'_>,
-    style: &StyleConfig,
-) -> Result<(), io::Error> {
-    use HighlightingSnippet::*;
-
-    match snip {
-        CommandName(s) => write!(writer, "{}", style.command_name.paint(s)),
-        Variable(s) => write!(writer, "{}", style.example_variable.paint(s)),
-        NormalCode(s) => write!(writer, "{}", style.example_code.paint(s)),
-        Description(s) => writeln!(writer, "  {}", style.description.paint(s)),
-        Text(s) => writeln!(writer, "  {}", style.example_text.paint(s)),
-        Linebreak => writeln!(writer),
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/main.rs b/src/main.rs
index be65e26..69011b9 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,12 +15,9 @@
 #![allow(clippy::similar_names)]
 #![allow(clippy::too_many_lines)]
 
-use std::fs::File;
-use std::io::BufRead;
-use std::io::BufReader;
+use std::env;
 use std::path::PathBuf;
 use std::process;
-use std::{env, io::Write};
 
 use ansi_term::{Color, Style};
 use app_dirs::AppInfo;
@@ -36,14 +33,14 @@ mod error;
 pub mod extensions;
 mod formatter;
 mod line_iterator;
+mod output;
 mod types;
 
 use crate::cache::{Cache, PageLookupResult};
 use crate::config::{get_config_dir, get_config_path, make_default_config, Config, MAX_CACHE_AGE};
 use crate::error::TealdeerError::ConfigError;
 use crate::extensions::Dedup;
-use crate::formatter::print_lines;
-use crate::line_iterator::LineIterator;
+use crate::output::print_page;
 use crate::types::{ColorOptions, OsType};
 
 const NAME: &str = "tealdeer";
@@ -78,43 +75,6 @@ struct Args {
     flag_language: Option<String>,
 }
 
-/// Print page by path
-fn print_page(
-    page: &PageLookupResult,
-    enable_markdown: bool,
-    config: &Config,
-) -> Result<(), String> {
-    let stdout = std::io::stdout();
-    let mut handle = stdout.lock();
-
-    for path in page.paths() {
-        let file = File::open(path).map_err(|msg| format!("Could not open file: {}", msg))?;
-        let reader = BufReader::new(file);
-
-        if enable_markdown {
-            // Print the raw markdown of the file.
-            for line in reader.lines() {
-                writeln!(handle, "{}", line.unwrap())
-                    .map_err(|_| "Could not write to stdout".to_string())?;
-            }
-        } else {
-            print_lines(
-                &mut handle,
-                LineIterator::new(reader),
-                &config.style,
-                !config.display.compact,
-            )
-            .map_err(|e| format!("Could not write to stdout: {}", e.message()))?;
-        };
-    }
-
-    handle
-        .flush()
-        .map_err(|_| "Could not flush stdout".to_string())?;
-
-    Ok(())
-}
-
 /// Set up display pager
 #[cfg(not(target_os = "windows"))]
 fn configure_pager() {
diff --git a/src/output.rs b/src/output.rs
new file mode 100644
index 0000000..cc3c7a0
--- /dev/null
+++ b/src/output.rs
@@ -0,0 +1,71 @@
+//! Functions for printing pages to the terminal
+
+use std::fs::File;
+use std::io::{self, BufRead, BufReader, Write};
+
+use crate::cache::PageLookupResult;
+use crate::config::{Config, StyleConfig};
+use crate::error::TealdeerError::WriteError;
+use crate::formatter::{highlight_lines, HighlightingSnippet};
+use crate::line_iterator::LineIterator;
+
+/// Print page by path
+pub fn print_page(
+    page: &PageLookupResult,
+    enable_markdown: bool,
+    config: &Config,
+) -> Result<(), String> {
+    let stdout = io::stdout();
+    let mut handle = stdout.lock();
+
+    for path in page.paths() {
+        let file = File::open(path).map_err(|msg| format!("Could not open file: {}", msg))?;
+        let reader = BufReader::new(file);
+
+        if enable_markdown {
+            // Print the raw markdown of the file.
+            for line in reader.lines() {
+                writeln!(handle, "{}", line.unwrap())
+                    .map_err(|_| "Could not write to stdout".to_string())?;
+            }
+        } else {
+            let mut yield_snippet = |snip: HighlightingSnippet<'_>| {
+                if snip.is_empty() {
+                    Ok(())
+                } else {
+                    print_snippet(&mut handle, snip, &config.style)
+                        .map_err(|e| WriteError(e.to_string()))
+                }
+            };
+            highlight_lines(
+                LineIterator::new(reader),
+                &mut yield_snippet,
+                !config.display.compact,
+            )
+            .map_err(|e| format!("Could not write to stdout: {}", e.message()))?;
+        };
+    }
+
+    handle
+        .flush()
+        .map_err(|_| "Could not flush stdout".to_string())?;
+
+    Ok(())
+}
+
+fn print_snippet(
+    writer: &mut impl Write,
+    snip: HighlightingSnippet<'_>,
+    style: &StyleConfig,
+) -> Result<(), io::Error> {
+    use HighlightingSnippet::*;
+
+    match snip {
+        CommandName(s) => write!(writer, "{}", style.command_name.paint(s)),
+        Variable(s) => write!(writer, "{}", style.example_variable.paint(s)),
+        NormalCode(s) => write!(writer, "{}", style.example_code.paint(s)),
+        Description(s) => writeln!(writer, "  {}", style.description.paint(s)),
+        Text(s) => writeln!(writer, "  {}", style.example_text.paint(s)),
+        Linebreak => writeln!(writer),
+    }
+}

From 743998a75fd398906826c01cf3050f7f158efdfc Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Wed, 2 Jun 2021 22:23:35 +0200
Subject: [PATCH 5/8] Reorder functions in `formatter.rs`

As someone reading this file for the first time, I would want to see the
only public and most general function first and find the specifics
further down instead of having to look for the "module entry" first.
---
 src/formatter.rs | 127 ++++++++++++++++++++++++-----------------------
 1 file changed, 64 insertions(+), 63 deletions(-)

diff --git a/src/formatter.rs b/src/formatter.rs
index 54d805d..af33ae9 100644
--- a/src/formatter.rs
+++ b/src/formatter.rs
@@ -6,6 +6,7 @@ use crate::types::LineType;
 use log::debug;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+/// Represents a snippet from a page from a specific highlighting class.
 pub enum HighlightingSnippet<'a> {
     CommandName(&'a str),
     Variable(&'a str),
@@ -26,69 +27,7 @@ impl<'a> HighlightingSnippet<'a> {
     }
 }
 
-/// Checks whether the characters right before and after the substring (given by half-open index interval) are whitespace (if they exist).
-fn is_freestanding_substring(surrouding: &str, substring: (usize, usize)) -> bool {
-    let (start, end) = substring;
-    // "okay" meaning <exists and is whitespace> or <doesn't exist>
-    let char_before_is_okay = surrouding[..start]
-        .chars()
-        .last()
-        .filter(|prev_char| !prev_char.is_whitespace())
-        .is_none();
-    let char_after_is_okay = surrouding[end..]
-        .chars()
-        .next()
-        .filter(|next_char| !next_char.is_whitespace())
-        .is_none();
-    char_before_is_okay && char_after_is_okay
-}
-
-/// Yields `NormalCode` and `CommandName` in alternating order according to the occurences of
-/// `command_name` in `segment`. Variables are not detected here, see `highlight_code`
-/// instead.
-fn highlight_code_segment<'a, E>(
-    command_name: &'a str,
-    mut segment: &'a str,
-    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
-) -> Result<(), E> {
-    if !command_name.is_empty() {
-        let mut search_start = 0;
-        while let Some(match_start) = segment.find_from(command_name, search_start) {
-            let match_end = match_start + command_name.len();
-            if is_freestanding_substring(segment, (match_start, match_end)) {
-                yield_snippet(HighlightingSnippet::NormalCode(&segment[..match_start]))?;
-                yield_snippet(HighlightingSnippet::CommandName(command_name))?;
-                segment = &segment[match_end..];
-                search_start = 0;
-            } else {
-                search_start = segment[match_start..]
-                    .char_indices()
-                    .nth(1)
-                    .map_or(segment.len(), |(i, _)| match_start + i);
-            }
-        }
-    }
-    yield_snippet(HighlightingSnippet::NormalCode(segment))?;
-    Ok(())
-}
-
-/// Highlight code examples including user variables in {{ curly braces }}.
-fn highlight_code<'a, E>(
-    command: &'a str,
-    text: &'a str,
-    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
-) -> Result<(), E> {
-    let variable_splits = text
-        .split("}}")
-        .map(|s| s.split_once("{{").unwrap_or((s, "")));
-    for (code_segment, variable) in variable_splits {
-        highlight_code_segment(&command, code_segment, yield_snippet)?;
-        yield_snippet(HighlightingSnippet::Variable(variable))?;
-    }
-    Ok(())
-}
-
-/// Print a token stream to an ANSI terminal.
+/// Parse the content of each line yielded by `lines` and yield `HighLightingSnippet`s accordingly.
 pub fn highlight_lines<L, F, E>(
     lines: L,
     yield_snippet: &mut F,
@@ -129,6 +68,68 @@ where
     Ok(())
 }
 
+/// Highlight code examples including user variables in {{ curly braces }}.
+fn highlight_code<'a, E>(
+    command: &'a str,
+    text: &'a str,
+    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
+) -> Result<(), E> {
+    let variable_splits = text
+        .split("}}")
+        .map(|s| s.split_once("{{").unwrap_or((s, "")));
+    for (code_segment, variable) in variable_splits {
+        highlight_code_segment(command, code_segment, yield_snippet)?;
+        yield_snippet(HighlightingSnippet::Variable(variable))?;
+    }
+    Ok(())
+}
+
+/// Yields `NormalCode` and `CommandName` in alternating order according to the occurences of
+/// `command_name` in `segment`. Variables are not detected here, see `highlight_code`
+/// instead.
+fn highlight_code_segment<'a, E>(
+    command_name: &'a str,
+    mut segment: &'a str,
+    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
+) -> Result<(), E> {
+    if !command_name.is_empty() {
+        let mut search_start = 0;
+        while let Some(match_start) = segment.find_from(command_name, search_start) {
+            let match_end = match_start + command_name.len();
+            if is_freestanding_substring(segment, (match_start, match_end)) {
+                yield_snippet(HighlightingSnippet::NormalCode(&segment[..match_start]))?;
+                yield_snippet(HighlightingSnippet::CommandName(command_name))?;
+                segment = &segment[match_end..];
+                search_start = 0;
+            } else {
+                search_start = segment[match_start..]
+                    .char_indices()
+                    .nth(1)
+                    .map_or(segment.len(), |(i, _)| match_start + i);
+            }
+        }
+    }
+    yield_snippet(HighlightingSnippet::NormalCode(segment))?;
+    Ok(())
+}
+
+/// Checks whether the characters right before and after the substring (given by half-open index interval) are whitespace (if they exist).
+fn is_freestanding_substring(surrouding: &str, substring: (usize, usize)) -> bool {
+    let (start, end) = substring;
+    // "okay" meaning <exists and is whitespace> or <doesn't exist>
+    let char_before_is_okay = surrouding[..start]
+        .chars()
+        .last()
+        .filter(|prev_char| !prev_char.is_whitespace())
+        .is_none();
+    let char_after_is_okay = surrouding[end..]
+        .chars()
+        .next()
+        .filter(|next_char| !next_char.is_whitespace())
+        .is_none();
+    char_before_is_okay && char_after_is_okay
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

From 22baa455b5f9e2e2e331771ba74b194648b03f34 Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Wed, 2 Jun 2021 22:31:05 +0200
Subject: [PATCH 6/8] Update inkscape-default.expected

---
 tests/inkscape-default.expected | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/inkscape-default.expected b/tests/inkscape-default.expected
index 7350de7..da909f2 100644
--- a/tests/inkscape-default.expected
+++ b/tests/inkscape-default.expected
@@ -4,29 +4,29 @@
 
   Open an SVG file in the Inkscape GUI:
 
-      inkscape filename.svg
+      inkscape filename.svg
 
   Export an SVG file into a bitmap with the default format (PNG) and the default resolution (90 DPI):
 
-      inkscape filename.svg -e filename.png
+      inkscape filename.svg -e filename.png
 
   Export an SVG file into a bitmap of 600x400 pixels (aspect ratio distortion may occur):
 
-      inkscape filename.svg -e filename.png -w 600 -h 400
+      inkscape filename.svg -e filename.png -w 600 -h 400
 
   Export a single object, given its ID, into a bitmap:
 
-      inkscape filename.svg -i id -e object.png
+      inkscape filename.svg -i id -e object.png
 
   Export an SVG document to PDF, converting all texts to paths:
 
-      inkscape filename.svg | inkscape | inkscape --export-pdf=inkscape.pdf | inkscape | inkscape --export-text-to-path
+      inkscape filename.svg | inkscape | inkscape --export-pdf=inkscape.pdf | inkscape | inkscape --export-text-to-path
 
   Duplicate the object with id="path123", rotate the duplicate 90 degrees, save the file, and quit Inkscape:
 
-      inkscape filename.svg --select=path123 --verb=EditDuplicate --verb=ObjectRotate90 --verb=FileSave --verb=FileQuit
+      inkscape filename.svg --select=path123 --verb=EditDuplicate --verb=ObjectRotate90 --verb=FileSave --verb=FileQuit
 
   Some invalid command just to test the correct highlighting of the command name:
 
-      inkscape --use-inkscape=v3.0 file
+      inkscape --use-inkscape=v3.0 file
 

From b216a63c64568dfb28a5a1fc7f1b4ef4fbe5d191 Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Wed, 2 Jun 2021 22:49:37 +0200
Subject: [PATCH 7/8] Add integration test for non-ascii page rendering

Although this did not fail on old master, it is still an important case
to test. Actually, this test doesn't even break when introducing some
errors that fail i18n unit tests, unless the command name itself is
non-ascii.
---
 tests/chmod.ru.expected | 32 ++++++++++++++++++++++++++++++++
 tests/chmod.ru.md       | 32 ++++++++++++++++++++++++++++++++
 tests/lib.rs            | 10 ++++++++++
 3 files changed, 74 insertions(+)
 create mode 100644 tests/chmod.ru.expected
 create mode 100644 tests/chmod.ru.md

diff --git a/tests/chmod.ru.expected b/tests/chmod.ru.expected
new file mode 100644
index 0000000..c7f92c2
--- /dev/null
+++ b/tests/chmod.ru.expected
@@ -0,0 +1,32 @@
+
+  Изменить права доступа файлу или папке.
+  Больше информации: <https://www.gnu.org/software/coreutils/chmod>.
+
+  Дать [u]пользователю, который владеет файлом, права на его [x]исполнение:
+
+      chmod u+x файл
+
+  Дать права [u]пользователю права [r]чтения и [w]записи в файл/папку:
+
+      chmod u+rw файл_или_папка
+
+  Убрать права на [x]исполнение у [g]группы:
+
+      chmod g-x файл
+
+  Дать [a]всем пользователям права на [r]чтение и [x]исполенеие:
+
+      chmod a+rx файл
+
+  Дать [o]другим (не из группы владельцев файлом) такие же права как и у [g]группы:
+
+      chmod o=g файл
+
+  Убрать все права у [o]других:
+
+      chmod o= файл
+
+  Изменить права рекурсивно, дав [g]группе и [o]другим возможность [w]записи в папку:
+
+      chmod -R g+w,o+w папка
+
diff --git a/tests/chmod.ru.md b/tests/chmod.ru.md
new file mode 100644
index 0000000..4b92329
--- /dev/null
+++ b/tests/chmod.ru.md
@@ -0,0 +1,32 @@
+# chmod
+
+> Изменить права доступа файлу или папке.
+> Больше информации: <https://www.gnu.org/software/coreutils/chmod>.
+
+- Дать [u]пользователю, который владеет файлом, права на его [x]исполнение:
+
+`chmod u+x {{файл}}`
+
+- Дать права [u]пользователю права [r]чтения и [w]записи в файл/папку:
+
+`chmod u+rw {{файл_или_папка}}`
+
+- Убрать права на [x]исполнение у [g]группы:
+
+`chmod g-x {{файл}}`
+
+- Дать [a]всем пользователям права на [r]чтение и [x]исполенеие:
+
+`chmod a+rx {{файл}}`
+
+- Дать [o]другим (не из группы владельцев файлом) такие же права как и у [g]группы:
+
+`chmod o=g {{файл}}`
+
+- Убрать все права у [o]других:
+
+`chmod o= {{файл}}`
+
+- Изменить права рекурсивно, дав [g]группе и [o]другим возможность [w]записи в папку:
+
+`chmod -R g+w,o+w {{папка}}`
diff --git a/tests/lib.rs b/tests/lib.rs
index 769901a..9a77373 100644
--- a/tests/lib.rs
+++ b/tests/lib.rs
@@ -362,6 +362,16 @@ fn test_rendering_color_never() {
     );
 }
 
+#[test]
+fn test_rendering_i18n() {
+    _test_correct_rendering(
+        include_str!("chmod.ru.md"),
+        "chmod.ru.md",
+        include_str!("chmod.ru.expected"),
+        "always",
+    );
+}
+
 /// An end-to-end integration test for rendering with custom syntax config.
 #[test]
 fn test_correct_rendering_with_config() {

From 3ac087ae415eab3d4f36e8afed3f0f3fa5a9137a Mon Sep 17 00:00:00 2001
From: Niklas Mohrin <dev@niklasmohrin.de>
Date: Wed, 8 Sep 2021 15:02:25 +0200
Subject: [PATCH 8/8] Final touches for formatting refactor

- Rename: `yield_snippet` => `process_snippet`
- Rename: `HighlightingSnippet` => `PageSnippet`
---
 src/formatter.rs | 48 ++++++++++++++++++++++++------------------------
 src/output.rs    | 10 +++++-----
 2 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/formatter.rs b/src/formatter.rs
index af33ae9..d1cfde4 100644
--- a/src/formatter.rs
+++ b/src/formatter.rs
@@ -6,8 +6,8 @@ use crate::types::LineType;
 use log::debug;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-/// Represents a snippet from a page from a specific highlighting class.
-pub enum HighlightingSnippet<'a> {
+/// Represents a snippet from a page of a specific highlighting class.
+pub enum PageSnippet<'a> {
     CommandName(&'a str),
     Variable(&'a str),
     NormalCode(&'a str),
@@ -16,9 +16,9 @@ pub enum HighlightingSnippet<'a> {
     Linebreak,
 }
 
-impl<'a> HighlightingSnippet<'a> {
+impl<'a> PageSnippet<'a> {
     pub fn is_empty(&self) -> bool {
-        use HighlightingSnippet::*;
+        use PageSnippet::*;
 
         match self {
             CommandName(s) | Variable(s) | NormalCode(s) | Description(s) | Text(s) => s.is_empty(),
@@ -30,19 +30,19 @@ impl<'a> HighlightingSnippet<'a> {
 /// Parse the content of each line yielded by `lines` and yield `HighLightingSnippet`s accordingly.
 pub fn highlight_lines<L, F, E>(
     lines: L,
-    yield_snippet: &mut F,
+    process_snippet: &mut F,
     keep_empty_lines: bool,
 ) -> Result<(), E>
 where
     L: Iterator<Item = LineType>,
-    F: for<'snip> FnMut(HighlightingSnippet<'snip>) -> Result<(), E>,
+    F: for<'snip> FnMut(PageSnippet<'snip>) -> Result<(), E>,
 {
     let mut command = String::new();
     for line in lines {
         match line {
             LineType::Empty => {
                 if keep_empty_lines {
-                    yield_snippet(HighlightingSnippet::Linebreak)?;
+                    process_snippet(PageSnippet::Linebreak)?;
                 }
             }
             LineType::Title(title) => {
@@ -53,18 +53,18 @@ where
                 command = title;
                 debug!("Detected command name: {}", &command);
             }
-            LineType::Description(text) => yield_snippet(HighlightingSnippet::Description(&text))?,
-            LineType::ExampleText(text) => yield_snippet(HighlightingSnippet::Text(&text))?,
+            LineType::Description(text) => process_snippet(PageSnippet::Description(&text))?,
+            LineType::ExampleText(text) => process_snippet(PageSnippet::Text(&text))?,
             LineType::ExampleCode(text) => {
-                yield_snippet(HighlightingSnippet::NormalCode("      "))?;
-                highlight_code(&command, &text, yield_snippet)?;
-                yield_snippet(HighlightingSnippet::Linebreak)?;
+                process_snippet(PageSnippet::NormalCode("      "))?;
+                highlight_code(&command, &text, process_snippet)?;
+                process_snippet(PageSnippet::Linebreak)?;
             }
 
             LineType::Other(text) => debug!("Unknown line type: {:?}", text),
         }
     }
-    yield_snippet(HighlightingSnippet::Linebreak)?;
+    process_snippet(PageSnippet::Linebreak)?;
     Ok(())
 }
 
@@ -72,14 +72,14 @@ where
 fn highlight_code<'a, E>(
     command: &'a str,
     text: &'a str,
-    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
+    process_snippet: &mut impl FnMut(PageSnippet<'a>) -> Result<(), E>,
 ) -> Result<(), E> {
     let variable_splits = text
         .split("}}")
         .map(|s| s.split_once("{{").unwrap_or((s, "")));
     for (code_segment, variable) in variable_splits {
-        highlight_code_segment(command, code_segment, yield_snippet)?;
-        yield_snippet(HighlightingSnippet::Variable(variable))?;
+        highlight_code_segment(command, code_segment, process_snippet)?;
+        process_snippet(PageSnippet::Variable(variable))?;
     }
     Ok(())
 }
@@ -90,15 +90,15 @@ fn highlight_code<'a, E>(
 fn highlight_code_segment<'a, E>(
     command_name: &'a str,
     mut segment: &'a str,
-    yield_snippet: &mut impl FnMut(HighlightingSnippet<'a>) -> Result<(), E>,
+    process_snippet: &mut impl FnMut(PageSnippet<'a>) -> Result<(), E>,
 ) -> Result<(), E> {
     if !command_name.is_empty() {
         let mut search_start = 0;
         while let Some(match_start) = segment.find_from(command_name, search_start) {
             let match_end = match_start + command_name.len();
             if is_freestanding_substring(segment, (match_start, match_end)) {
-                yield_snippet(HighlightingSnippet::NormalCode(&segment[..match_start]))?;
-                yield_snippet(HighlightingSnippet::CommandName(command_name))?;
+                process_snippet(PageSnippet::NormalCode(&segment[..match_start]))?;
+                process_snippet(PageSnippet::CommandName(command_name))?;
                 segment = &segment[match_end..];
                 search_start = 0;
             } else {
@@ -109,7 +109,7 @@ fn highlight_code_segment<'a, E>(
             }
         }
     }
-    yield_snippet(HighlightingSnippet::NormalCode(segment))?;
+    process_snippet(PageSnippet::NormalCode(segment))?;
     Ok(())
 }
 
@@ -133,7 +133,7 @@ fn is_freestanding_substring(surrouding: &str, substring: (usize, usize)) -> boo
 #[cfg(test)]
 mod tests {
     use super::*;
-    use HighlightingSnippet::*;
+    use PageSnippet::*;
 
     #[test]
     fn test_is_freestanding_substring() {
@@ -160,16 +160,16 @@ mod tests {
         ));
     }
 
-    fn run<'a>(cmd: &'a str, segment: &'a str) -> Vec<HighlightingSnippet<'a>> {
+    fn run<'a>(cmd: &'a str, segment: &'a str) -> Vec<PageSnippet<'a>> {
         let mut yielded = Vec::new();
-        let mut yield_snippet = |snip: HighlightingSnippet<'a>| {
+        let mut process_snippet = |snip: PageSnippet<'a>| {
             if !snip.is_empty() {
                 yielded.push(snip);
             }
             Ok::<(), ()>(())
         };
 
-        highlight_code_segment(cmd, segment, &mut yield_snippet)
+        highlight_code_segment(cmd, segment, &mut process_snippet)
             .expect("highlight code segment failed");
         yielded
     }
diff --git a/src/output.rs b/src/output.rs
index cc3c7a0..25b9bc3 100644
--- a/src/output.rs
+++ b/src/output.rs
@@ -6,7 +6,7 @@ use std::io::{self, BufRead, BufReader, Write};
 use crate::cache::PageLookupResult;
 use crate::config::{Config, StyleConfig};
 use crate::error::TealdeerError::WriteError;
-use crate::formatter::{highlight_lines, HighlightingSnippet};
+use crate::formatter::{highlight_lines, PageSnippet};
 use crate::line_iterator::LineIterator;
 
 /// Print page by path
@@ -29,7 +29,7 @@ pub fn print_page(
                     .map_err(|_| "Could not write to stdout".to_string())?;
             }
         } else {
-            let mut yield_snippet = |snip: HighlightingSnippet<'_>| {
+            let mut process_snippet = |snip: PageSnippet<'_>| {
                 if snip.is_empty() {
                     Ok(())
                 } else {
@@ -39,7 +39,7 @@ pub fn print_page(
             };
             highlight_lines(
                 LineIterator::new(reader),
-                &mut yield_snippet,
+                &mut process_snippet,
                 !config.display.compact,
             )
             .map_err(|e| format!("Could not write to stdout: {}", e.message()))?;
@@ -55,10 +55,10 @@ pub fn print_page(
 
 fn print_snippet(
     writer: &mut impl Write,
-    snip: HighlightingSnippet<'_>,
+    snip: PageSnippet<'_>,
     style: &StyleConfig,
 ) -> Result<(), io::Error> {
-    use HighlightingSnippet::*;
+    use PageSnippet::*;
 
     match snip {
         CommandName(s) => write!(writer, "{}", style.command_name.paint(s)),