diff --git a/src/config.rs b/src/config.rs index b31f422..7c29ed4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -10,6 +10,7 @@ use std::{ use anyhow::{anyhow, bail, ensure, Context, Result}; use app_dirs::{get_app_root, AppDataType}; use clap::ValueEnum; +use log::info; use serde::Serialize as _; use serde_derive::{Deserialize, Serialize}; use yansi::{Color, Style}; @@ -429,6 +430,11 @@ fn get_languages<'a>( let mut lang_list = Vec::new(); for locale in locales { + if !locale.is_ascii() { + info!("Skipping non-ASCII locale string: {}", locale); + continue; + } + // Language plus country code (e.g. `en_US`) if locale.len() >= 5 && locale.chars().nth(2) == Some('_') { lang_list.push(Language(&locale[..5])); diff --git a/src/formatter.rs b/src/formatter.rs index 1cdc681..c0d3a91 100644 --- a/src/formatter.rs +++ b/src/formatter.rs @@ -179,7 +179,11 @@ fn find_marker(s: &str, marker: &str, forbidden_prefix: &str) -> Option { let overlaps_with_prefix = (forbidden_prefix.len() <= marker_index + 1) && { let prefix_start = marker_index + 1 - forbidden_prefix.len(); - &s[prefix_start..=marker_index] == forbidden_prefix + // NOTE: The indices might not be valid character offsets, so we should do this + // comparison on raw bytes. If prefix_start is indeed not a character offset than the + // comparison is guaranteed to return false because forbidden_prefix[0] definitely _is_ + // the start of a (single byte, ASCII) character. + &s.as_bytes()[prefix_start..=marker_index] == forbidden_prefix.as_bytes() }; if !overlaps_with_prefix { return Some(marker_index); @@ -446,5 +450,12 @@ mod tests { ], ); } + + #[test] + /// Regression test for https://github.com/tealdeer-rs/tealdeer/issues/473 + fn prefix_check_character_boundary() { + assert_eq!("Ä".len(), 2); + assert_eq!(run("", r#"Äxx{{x}}"#), [NormalCode("Äxx"), Variable("x")],); + } } }