From e8b1c9e801cc07348e2d7e550fd5a6b473479bbd Mon Sep 17 00:00:00 2001 From: Niklas Mohrin Date: Wed, 2 Jun 2021 21:04:05 +0200 Subject: [PATCH] Rename `Tokenizer` to `LineIterator` This also replaces the `next_token` method with the `next` method from the `Iterator` trait. --- src/formatter.rs | 14 +++--- src/{tokenizer.rs => line_iterator.rs} | 68 +++++++++++++------------- src/main.rs | 8 ++- 3 files changed, 44 insertions(+), 46 deletions(-) rename src/{tokenizer.rs => line_iterator.rs} (53%) diff --git a/src/formatter.rs b/src/formatter.rs index dd7335d..9a26c1d 100644 --- a/src/formatter.rs +++ b/src/formatter.rs @@ -1,13 +1,12 @@ //! Functions related to formatting and printing lines from a `Tokenizer`. -use std::io::{self, BufRead, Write}; +use std::io::{self, Write}; use log::debug; use crate::config::StyleConfig; use crate::error::TealdeerError::{self, WriteError}; use crate::extensions::FindFrom; -use crate::tokenizer::Tokenizer; use crate::types::LineType; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -94,15 +93,14 @@ fn highlight_code<'a, E>( } /// Print a token stream to an ANSI terminal. -pub fn print_lines( +pub fn print_lines( writer: &mut T, - tokenizer: &mut Tokenizer, + lines: impl Iterator, style: &StyleConfig, keep_empty_lines: bool, ) -> Result<(), TealdeerError> where T: Write, - R: BufRead, { let mut command = String::new(); let mut yield_snippet = |snip: HighlightingSnippet<'_>| { @@ -112,8 +110,8 @@ where print_snippet(writer, snip, style).map_err(|e| WriteError(e.to_string())) } }; - while let Some(token) = tokenizer.next_token() { - match token { + for line in lines { + match line { LineType::Empty => { if keep_empty_lines { yield_snippet(HighlightingSnippet::Linebreak)?; @@ -123,7 +121,7 @@ where debug!("Ignoring title"); // This is safe as long as the parsed title is only the command, - // and tokenizer yields values in order of appearance. + // and the iterator yields values in order of appearance. command = title; debug!("Detected command name: {}", &command); } diff --git a/src/tokenizer.rs b/src/line_iterator.rs similarity index 53% rename from src/tokenizer.rs rename to src/line_iterator.rs index f9b3837..c1fbb0f 100644 --- a/src/tokenizer.rs +++ b/src/line_iterator.rs @@ -1,4 +1,4 @@ -//! Code to tokenize a `BufRead` instance into an iterator of `LineType`s. +//! Code to split a `BufRead` instance into an iterator of `LineType`s. use std::io::BufRead; @@ -15,13 +15,13 @@ pub enum TldrFormat { V2, } -/// A tokenizer is initialized with a `BufReader` instance that contains the -/// entire Tldr page. It then returns tokens as `Option`. +/// A `LineIterator` is initialized with a `BufReader` instance that contains the +/// entire Tldr page. It then implements `Iterator`. #[derive(Debug)] -pub struct Tokenizer { +pub struct LineIterator { /// An instance of `R: BufRead`. reader: R, - /// Whether the first line has already been tokenized or not. + /// Whether the first line has already been processed or not. first_line: bool, /// Buffer for the current line. Used internally. current_line: String, @@ -29,7 +29,7 @@ pub struct Tokenizer { format: TldrFormat, } -impl Tokenizer +impl LineIterator where R: BufRead, { @@ -41,38 +41,40 @@ where format: TldrFormat::Undecided, } } +} - pub fn next_token(&mut self) -> Option { +impl Iterator for LineIterator { + type Item = LineType; + + fn next(&mut self) -> Option { self.current_line.clear(); let bytes_read = self.reader.read_line(&mut self.current_line); match bytes_read { Ok(0) => None, Err(e) => { - warn!("Could not read line from token reader: {:?}", e); + warn!("Could not read line from reader: {:?}", e); None } Ok(_) => { // Handle new titles - if self.first_line && !self.current_line.starts_with('#') { - // It's the new format! Drop next line. - // (Hmm, is there a way to do this without an allocation?) - let mut devnull = String::new(); - if let Err(e) = self.reader.read_line(&mut devnull) { - warn!("Could not read line from token reader: {:?}", e); - return None; - } - self.first_line = false; - self.format = TldrFormat::V2; - return Some(LineType::Title(self.current_line.trim_end().to_string())); - } - if self.first_line { - // Clear `first_line` flag - self.first_line = false; - - // It's the old format. - self.format = TldrFormat::V1; + if self.current_line.starts_with('#') { + // It's the old format. + self.format = TldrFormat::V1; + } else { + // It's the new format! Drop next line. + // (Hmm, is there a way to do this without an allocation?) + let mut devnull = String::new(); + if let Err(e) = self.reader.read_line(&mut devnull) { + warn!("Could not read line from reader: {:?}", e); + return None; + } + self.first_line = false; + self.format = TldrFormat::V2; + return Some(LineType::Title(self.current_line.trim_end().to_string())); + } } + self.first_line = false; // Convert line to a `LineType` instance match self.format { @@ -87,26 +89,26 @@ where #[cfg(test)] mod test { - use super::Tokenizer; + use super::LineIterator; use crate::types::LineType; #[test] fn test_first_line_old_format() { let input = "# The Title\n\n"; - let mut tokenizer = Tokenizer::new(input.as_bytes()); - let title = tokenizer.next_token().unwrap(); + let mut lines = LineIterator::new(input.as_bytes()); + let title = lines.next().unwrap(); assert_eq!(title, LineType::Title("The Title".to_string())); - let empty = tokenizer.next_token().unwrap(); + let empty = lines.next().unwrap(); assert_eq!(empty, LineType::Empty); } #[test] fn test_first_line_new_format() { let input = "The Title\n=========\n\n"; - let mut tokenizer = Tokenizer::new(input.as_bytes()); - let title = tokenizer.next_token().unwrap(); + let mut lines = LineIterator::new(input.as_bytes()); + let title = lines.next().unwrap(); assert_eq!(title, LineType::Title("The Title".to_string())); - let empty = tokenizer.next_token().unwrap(); + let empty = lines.next().unwrap(); assert_eq!(empty, LineType::Empty); } } diff --git a/src/main.rs b/src/main.rs index ffdfb92..be65e26 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ mod config; mod error; pub mod extensions; mod formatter; -mod tokenizer; +mod line_iterator; mod types; use crate::cache::{Cache, PageLookupResult}; @@ -43,7 +43,7 @@ use crate::config::{get_config_dir, get_config_path, make_default_config, Config use crate::error::TealdeerError::ConfigError; use crate::extensions::Dedup; use crate::formatter::print_lines; -use crate::tokenizer::Tokenizer; +use crate::line_iterator::LineIterator; use crate::types::{ColorOptions, OsType}; const NAME: &str = "tealdeer"; @@ -98,11 +98,9 @@ fn print_page( .map_err(|_| "Could not write to stdout".to_string())?; } } else { - // Create tokenizer and print output - let mut tokenizer = Tokenizer::new(reader); print_lines( &mut handle, - &mut tokenizer, + LineIterator::new(reader), &config.style, !config.display.compact, )