Merge 4686c84bac into e5cb757278

main.go

@@ -278,6 +278,7 @@ func executeCLI(cmd *cobra.Command, src *source, w io.Writer) error {
 		return fmt.Errorf("unable to read from reader: %w", err)
 	}
 
+	b = utils.DecodeUTF16BOM(b)
 	b = utils.RemoveFrontmatter(b)
 
 	// render

ui/stash.go

@@ -12,6 +12,7 @@ import (
 	"github.com/charmbracelet/bubbles/spinner"
 	"github.com/charmbracelet/bubbles/textinput"
 	tea "github.com/charmbracelet/bubbletea"
+	"github.com/charmbracelet/glow/v2/utils"
 	"github.com/charmbracelet/lipgloss"
 	"github.com/charmbracelet/log"
 	"github.com/muesli/reflow/ansi"
@@ -860,7 +861,7 @@ func loadLocalMarkdown(md *markdown) tea.Cmd {
 			log.Debug("error reading local file", "error", err)
 			return errMsg{err}
 		}
-		md.Body = string(data)
+		md.Body = string(utils.DecodeUTF16BOM(data))
 		return fetchedMarkdownMsg(md)
 	}
 }

ui/ui.go

@@ -195,7 +195,7 @@ func (m model) Init() tea.Cmd {
 			log.Error("unable to read file", "file", m.common.cfg.Path, "error", err)
 			return func() tea.Msg { return errMsg{err} }
 		}
-		body := string(utils.RemoveFrontmatter(content))
+		body := string(utils.RemoveFrontmatter(utils.DecodeUTF16BOM(content)))
 		cmds = append(cmds, renderWithGlamour(m.pager, body))
 	}
 

utils/encoding.go

@@ -0,0 +1,31 @@
+package utils
+
+import (
+	"encoding/binary"
+	"unicode/utf16"
+)
+
+// DecodeUTF16BOM converts UTF-16 text with a byte-order mark to UTF-8.
+// Data without a UTF-16 BOM, including malformed UTF-16 with an odd payload,
+// is returned unchanged.
+func DecodeUTF16BOM(data []byte) []byte {
+	if len(data) < 2 || (len(data)-2)%2 != 0 {
+		return data
+	}
+
+	var order binary.ByteOrder
+	switch {
+	case data[0] == 0xff && data[1] == 0xfe:
+		order = binary.LittleEndian
+	case data[0] == 0xfe && data[1] == 0xff:
+		order = binary.BigEndian
+	default:
+		return data
+	}
+
+	codeUnits := make([]uint16, (len(data)-2)/2)
+	for i := range codeUnits {
+		codeUnits[i] = order.Uint16(data[2+i*2:])
+	}
+	return []byte(string(utf16.Decode(codeUnits)))
+}

utils/encoding_test.go

@@ -0,0 +1,40 @@
+package utils
+
+import "testing"
+
+func TestDecodeUTF16BOM(t *testing.T) {
+	tests := []struct {
+		name string
+		data []byte
+		want string
+	}{
+		{
+			name: "little endian",
+			data: []byte{0xff, 0xfe, '#', 0, ' ', 0, 'T', 0, 'i', 0, 't', 0, 'l', 0, 'e', 0},
+			want: "# Title",
+		},
+		{
+			name: "big endian",
+			data: []byte{0xfe, 0xff, 0, '#', 0, ' ', 0, 'T', 0, 'i', 0, 't', 0, 'l', 0, 'e'},
+			want: "# Title",
+		},
+		{
+			name: "utf8 remains unchanged",
+			data: []byte("# Title"),
+			want: "# Title",
+		},
+		{
+			name: "odd utf16 payload remains unchanged",
+			data: []byte{0xff, 0xfe, '#'},
+			want: string([]byte{0xff, 0xfe, '#'}),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := string(DecodeUTF16BOM(tt.data)); got != tt.want {
+				t.Fatalf("DecodeUTF16BOM() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}