Fix off-boundary string access in formatter (#474)

Closes #473

This regression was introduced in
593e9309b9 (#414) and leads to a panic
when trying to display pages where characters line up like in the issue
or the test. I checked other places and found that a similar panic could
occur when parsing language strings, so I added code to ignore them
instead.

- Add regression test
- Fix prefix check
- Skip non-ASCII locales

src/config.rs

@@ -10,6 +10,7 @@ use std::{
 use anyhow::{anyhow, bail, ensure, Context, Result};
 use app_dirs::{get_app_root, AppDataType};
 use clap::ValueEnum;
+use log::info;
 use serde::Serialize as _;
 use serde_derive::{Deserialize, Serialize};
 use yansi::{Color, Style};
@@ -429,6 +430,11 @@ fn get_languages<'a>(
 
     let mut lang_list = Vec::new();
     for locale in locales {
+        if !locale.is_ascii() {
+            info!("Skipping non-ASCII locale string: {}", locale);
+            continue;
+        }
+
         // Language plus country code (e.g. `en_US`)
         if locale.len() >= 5 && locale.chars().nth(2) == Some('_') {
             lang_list.push(Language(&locale[..5]));

src/formatter.rs

@@ -179,7 +179,11 @@ fn find_marker(s: &str, marker: &str, forbidden_prefix: &str) -> Option<usize> {
 
         let overlaps_with_prefix = (forbidden_prefix.len() <= marker_index + 1) && {
             let prefix_start = marker_index + 1 - forbidden_prefix.len();
-            &s[prefix_start..=marker_index] == forbidden_prefix
+            // NOTE: The indices might not be valid character offsets, so we should do this
+            // comparison on raw bytes. If prefix_start is indeed not a character offset than the
+            // comparison is guaranteed to return false because forbidden_prefix[0] definitely _is_
+            // the start of a (single byte, ASCII) character.
+            &s.as_bytes()[prefix_start..=marker_index] == forbidden_prefix.as_bytes()
         };
         if !overlaps_with_prefix {
             return Some(marker_index);
@@ -446,5 +450,12 @@ mod tests {
                 ],
             );
         }
+
+        #[test]
+        /// Regression test for https://github.com/tealdeer-rs/tealdeer/issues/473
+        fn prefix_check_character_boundary() {
+            assert_eq!("Ä".len(), 2);
+            assert_eq!(run("", r#"Äxx{{x}}"#), [NormalCode("Äxx"), Variable("x")],);
+        }
     }
 }