Merge pull request #32 from dbrgn/new-format

Support new format. See https://github.com/tldr-pages/tldr/pull/958.
This commit is contained in:
Danilo Bargen 2016-11-21 09:52:33 +01:00 committed by GitHub
commit a44b5ffcc5
10 changed files with 274 additions and 11 deletions

View file

@ -7,7 +7,7 @@ addons:
- libelf-dev
- libdw-dev
rust:
- 1.6.0
- 1.9.0
- stable
- beta
- nightly

47
Cargo.lock generated
View file

@ -371,3 +371,50 @@ name = "xdg"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum aho-corasick 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2b3fb52b09c1710b961acb35390d514be82e4ac96a9969a8e38565a29b878dc9"
"checksum ansi_term 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c877397e09fec7a240af5fa74ad0124054b8066149d6544cd1ace93f8de3be68"
"checksum clippy 0.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "047e91f2cb238688731c03d9b17837249fc3ba3086feb5443d95f68b8ab48145"
"checksum clippy_lints 0.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "665fffe8ccc4384a4d817f2fea3af1e7a7360c4cee494b6a543cbf19aac4019d"
"checksum curl 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "11d108a51783e376b51b481eb67a1f408bad1135a992d1373ae300aa510133e9"
"checksum curl-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3da1d4b92dc22964e4b098c9e5863abfb9126d2c619bbeefb7eaa4ae63adbc5"
"checksum docopt 0.6.82 (registry+https://github.com/rust-lang/crates.io-index)" = "8f20016093b4e545dccf6ad4a01099de0b695f9bc99b08210e68f6425db2d37d"
"checksum env_logger 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "82dcb9ceed3868a03b335657b85a159736c961900f7e7747d3b0b97b9ccb5ccb"
"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
"checksum flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "3eeb481e957304178d2e782f2da1257f1434dfecbae883bafb61ada2a9fea3bb"
"checksum gcc 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "dcb000abd6df9df4c637f75190297ebe56c1d7e66b56bbf3b4aa7aece15f61a2"
"checksum gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0912515a8ff24ba900422ecda800b52f4016a56251922d397c576bf92c690518"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "49247ec2a285bb3dcb23cbd9c35193c025e7251bfce77c1d5da97e6362dffe7f"
"checksum libc 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "39dfaaa0f4da0f1a06876c5d94329d739ad0150868069cc235f1ddf80a0480e7"
"checksum libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "cbc058951ab6a3ef35ca16462d7642c4867e6403520811f28537a4e2f2db3e71"
"checksum libz-sys 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ab80d82d16acfbf5c0e731297db302378896c0cffbc428dddc61d995a82e15cf"
"checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054"
"checksum matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "15305656809ce5a4805b1ff2946892810992197ce1270ff79baded852187942e"
"checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
"checksum miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d1f4d337a01c32e1f2122510fed46393d53ca35a7f429cb0450abaedfa3ed54"
"checksum nom 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b8c256fd9471521bcb84c3cdba98921497f1a331cbc15b8030fc63b82050ce"
"checksum openssl-sys 0.7.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ac5e9d911dd4c3202bbf4139b73bc7a1231f7d0a39432c6f893745f0e04120"
"checksum pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8cee804ecc7eaf201a4a207241472cc870e825206f6c031e3ee2a72fa425f2fa"
"checksum pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "61c9231d31aea845007443d62fcbb58bb6949ab9c18081ee1e09920e0cf1118b"
"checksum quine-mc_cluskey 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6683b0e23d80813b1a535841f0048c1537d3f86d63c999e8373b39a9b0eb74a"
"checksum rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "2791d88c6defac799c3f20d74f094ca33b9332612d9aef9078519c82e4fe04a5"
"checksum regex 0.1.73 (registry+https://github.com/rust-lang/crates.io-index)" = "56b7ee9f764ecf412c6e2fff779bca4b22980517ae335a21aeaf4e32625a5df2"
"checksum regex-syntax 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "31040aad7470ad9d8c46302dcffba337bb4289ca5da2e3cd6e37b64109a85199"
"checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
"checksum semver 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2d5b7638a1f03815d94e88cb3b3c08e87f0db4d683ef499d1836aaf70a45623f"
"checksum strsim 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4d73a2c36a4d095ed1a6df5cbeac159863173447f7a82b3f4757426844ab825"
"checksum tar 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a05f62bf3862b1725b03059def56375a32178ee2c99dce2fb745f004b8bc82dc"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
"checksum thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03"
"checksum thread_local 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "55dd963dbaeadc08aa7266bf7f91c3154a7805e32bb94b820b769d2ef3b4744d"
"checksum time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "3c7ec6d62a20df54e07ab3b78b9a3932972f4b7981de295563686849eb3989af"
"checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796"
"checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"
"checksum user32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ef4711d107b21b410a3a974b1204d9accc8b10dad75d8324b5d755de1617d47"
"checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
"checksum walkdir 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "7ad450634b9022aeb0e8e7f1c79c1ded92d0fc5bee831033d148479771bd218d"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum xattr 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "fa3838ce46f489be85e47cb3b83e72c7be8d3577fd20c291877733beda97fffc"
"checksum xdg 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77b831a5ba77110f438f0ac5583aafeb087f70432998ba6b7dcb1d32185db453"

View file

@ -64,7 +64,7 @@ If you're an Arch Linux user, you can also install the package from the AUR:
## Building
tealdeer requires at least Rust 1.6.
tealdeer requires at least Rust 1.9.
Debug build with logging enabled:

View file

@ -29,8 +29,8 @@ pub fn print_lines<R>(tokenizer: &mut Tokenizer<R>) where R: BufRead {
LineType::Empty => println!(""),
LineType::Title(_) => debug!("Ignoring title"),
LineType::Description(text) => println!(" {}", text),
LineType::ExampleText(text) => println!(" {}", Colour::Green.paint(format!("- {}", text))),
LineType::ExampleCode(text) => println!(" {}", &format_braces(&text)),
LineType::ExampleText(text) => println!(" {}", Colour::Green.paint(text)),
LineType::ExampleCode(text) => println!(" {}", &format_braces(&text)),
LineType::Other(text) => debug!("Unknown line type: {:?}", text),
}
}

View file

@ -4,17 +4,37 @@ use std::io::BufRead;
use types::LineType;
#[derive(Debug, PartialEq, Eq)]
pub enum TldrFormat {
/// Not yet clear
Undecided,
/// The original format
V1,
/// The new format (see https://github.com/tldr-pages/tldr/pull/958)
V2,
}
/// A tokenizer is initialized with a BufReader instance that contains the
/// entire Tldr page. It then returns tokens as `Option<LineType>`.
#[derive(Debug)]
pub struct Tokenizer<R: BufRead> {
/// An instance of `R: BufRead`.
reader: R,
/// Whether the first line has already been tokenized or not.
first_line: bool,
/// Buffer for the current line. Used internally.
current_line: String,
/// The tldr page format.
format: TldrFormat,
}
impl<R> Tokenizer<R> where R: BufRead {
pub fn new(reader: R) -> Tokenizer<R> {
Tokenizer {
reader: reader,
first_line: true,
current_line: String::new(),
format: TldrFormat::Undecided,
}
}
@ -23,8 +43,66 @@ impl<R> Tokenizer<R> where R: BufRead {
let bytes_read = self.reader.read_line(&mut self.current_line);
match bytes_read {
Ok(0) => None,
Err(e) => { warn!("Could not read line from token reader: {:?}", e); None},
Ok(_) => Some(LineType::from(&self.current_line[..])),
Err(e) => {
warn!("Could not read line from token reader: {:?}", e);
None
},
Ok(_) => {
// Handle new titles
if self.first_line && !self.current_line.starts_with("#") {
// It's the new format! Drop next line.
// (Hmm, is there a way to do this without an allocation?)
let mut devnull = String::new();
if let Err(e) = self.reader.read_line(&mut devnull) {
warn!("Could not read line from token reader: {:?}", e);
return None;
}
self.first_line = false;
self.format = TldrFormat::V2;
return Some(LineType::Title(self.current_line.trim_right().to_string()));
}
if self.first_line {
// Clear `first_line` flag
self.first_line = false;
// It's the old format.
self.format = TldrFormat::V1;
}
// Convert line to a `LineType` instance
match self.format {
TldrFormat::V1 => Some(LineType::from_v1(&self.current_line[..])),
TldrFormat::V2 => Some(LineType::from(&self.current_line[..])),
TldrFormat::Undecided => panic!("Could not determine page format version"),
}
}
}
}
}
#[cfg(test)]
mod test {
use super::Tokenizer;
use types::LineType;
#[test]
fn test_first_line_old_format() {
let input = "# The Title\n\n";
let mut tokenizer = Tokenizer::new(input.as_bytes());
let title = tokenizer.next_token().unwrap();
assert_eq!(title, LineType::Title("The Title".to_string()));
let empty = tokenizer.next_token().unwrap();
assert_eq!(empty, LineType::Empty);
}
#[test]
fn test_first_line_new_format() {
let input = "The Title\n=========\n\n";
let mut tokenizer = Tokenizer::new(input.as_bytes());
let title = tokenizer.next_token().unwrap();
assert_eq!(title, LineType::Title("The Title".to_string()));
let empty = tokenizer.next_token().unwrap();
assert_eq!(empty, LineType::Empty);
}
}

View file

@ -44,8 +44,24 @@ pub enum LineType {
}
impl<'a> From<&'a str> for LineType {
/// Convert a string slice to a LineType. Newlines and whitespace are trimmed.
/// Convert a string slice to a LineType. Newlines and trailing whitespace are trimmed.
fn from(line: &'a str) -> LineType {
let trimmed: &str = line.trim_right();
let mut chars = trimmed.chars();
match chars.next() {
None => LineType::Empty,
Some('#') => LineType::Title(trimmed.trim_left_matches(|chr: char| chr == '#' || chr.is_whitespace()).into()),
Some('>') => LineType::Description(trimmed.trim_left_matches(|chr: char| chr == '>' || chr.is_whitespace()).into()),
Some(' ') => LineType::ExampleCode(trimmed.trim_left_matches(|chr: char| chr.is_whitespace()).into()),
_ => LineType::ExampleText(trimmed.into()),
}
}
}
impl LineType {
/// Support for old format.
/// TODO: Remove once old format has been phased out!
pub fn from_v1(line: &str) -> LineType {
let trimmed = line.trim();
let mut chars = trimmed.chars();
match chars.next() {
@ -93,9 +109,7 @@ mod test {
assert_eq!(LineType::from(" \n \r"), LineType::Empty);
assert_eq!(LineType::from("# Hello there"), LineType::Title("Hello there".into()));
assert_eq!(LineType::from("> tis a description \n"), LineType::Description("tis a description".into()));
assert_eq!(LineType::from("- some command"), LineType::ExampleText("some command".into()));
assert_eq!(LineType::from("`$ cargo run`"), LineType::ExampleCode("$ cargo run".into()));
assert_eq!(LineType::from("`$ cargo run"), LineType::Other("`$ cargo run".into()));
assert_eq!(LineType::from("jkl\u{f6}"), LineType::Other("jkl\u{f6}".into()));
assert_eq!(LineType::from("some command "), LineType::ExampleText("some command".into()));
assert_eq!(LineType::from(" $ cargo run "), LineType::ExampleCode("$ cargo run".into()));
}
}

28
tests/inkscape-v1.md Normal file
View file

@ -0,0 +1,28 @@
# inkscape
> An SVG (Scalable Vector Graphics) editing program.
> Use -z to not open the GUI and only process files in the console.
- Open an SVG file in the Inkscape GUI:
`inkscape {{filename.svg}}`
- Export an SVG file into a bitmap with the default format (PNG) and the default resolution (90 DPI):
`inkscape {{filename.svg}} -e {{filename.png}}`
- Export an SVG file into a bitmap of 600x400 pixels (aspect ratio distortion may occur):
`inkscape {{filename.svg}} -e {{filename.png}} -w {{600}} -h {{400}}`
- Export a single object, given its ID, into a bitmap:
`inkscape {{filename.svg}} -i {{id}} -e {{object.png}}`
- Export an SVG document to PDF, converting all texts to paths:
`inkscape {{filename.svg}} --export-pdf={{filename.pdf}} --export-text-to-path`
- Duplicate the object with id="path123", rotate the duplicate 90 degrees, save the file, and quit Inkscape:
`inkscape {{filename.svg}} --select=path123 --verb=EditDuplicate --verb=ObjectRotate90 --verb=FileSave --verb=FileQuit`

29
tests/inkscape-v2.md Normal file
View file

@ -0,0 +1,29 @@
inkscape
========
> An SVG (Scalable Vector Graphics) editing program.
> Use -z to not open the GUI and only process files in the console.
Open an SVG file in the Inkscape GUI:
inkscape {{filename.svg}}
Export an SVG file into a bitmap with the default format (PNG) and the default resolution (90 DPI):
inkscape {{filename.svg}} -e {{filename.png}}
Export an SVG file into a bitmap of 600x400 pixels (aspect ratio distortion may occur):
inkscape {{filename.svg}} -e {{filename.png}} -w {{600}} -h {{400}}
Export a single object, given its ID, into a bitmap:
inkscape {{filename.svg}} -i {{id}} -e {{object.png}}
Export an SVG document to PDF, converting all texts to paths:
inkscape {{filename.svg}} --export-pdf={{filename.pdf}} --export-text-to-path
Duplicate the object with id="path123", rotate the duplicate 90 degrees, save the file, and quit Inkscape:
inkscape {{filename.svg}} --select=path123 --verb=EditDuplicate --verb=ObjectRotate90 --verb=FileSave --verb=FileQuit

28
tests/inkscape.expected Normal file
View file

@ -0,0 +1,28 @@
An SVG (Scalable Vector Graphics) editing program.
Use -z to not open the GUI and only process files in the console.
Open an SVG file in the Inkscape GUI:
inkscape filename.svg
Export an SVG file into a bitmap with the default format (PNG) and the default resolution (90 DPI):
inkscape filename.svg -e filename.png
Export an SVG file into a bitmap of 600x400 pixels (aspect ratio distortion may occur):
inkscape filename.svg -e filename.png -w 600 -h 400
Export a single object, given its ID, into a bitmap:
inkscape filename.svg -i id -e object.png
Export an SVG document to PDF, converting all texts to paths:
inkscape filename.svg --export-pdf=filename.pdf --export-text-to-path
Duplicate the object with id="path123", rotate the duplicate 90 degrees, save the file, and quit Inkscape:
inkscape filename.svg --select=path123 --verb=EditDuplicate --verb=ObjectRotate90 --verb=FileSave --verb=FileQuit

View file

@ -3,6 +3,8 @@
extern crate tempdir;
use std::env;
use std::fs::File;
use std::io::{Read, BufReader};
use std::path::PathBuf;
use std::process::Command;
@ -11,6 +13,7 @@ use tempdir::TempDir;
struct TestEnv {
cache_dir: TempDir,
bin_path: PathBuf,
pub tests_path: PathBuf,
}
impl TestEnv {
@ -23,9 +26,16 @@ impl TestEnv {
let bin_dir = lib_path.parent().unwrap();
let bin_path = bin_dir.join("tldr");
// Copy test files
let tests_path = bin_dir.parent()
.and_then(|d| d.parent())
.map(|d| d.join("tests"))
.expect("Could not find tests directory path");
TestEnv {
cache_dir: dir,
bin_path: bin_path,
tests_path: tests_path,
}
}
@ -75,3 +85,32 @@ fn test_update_cache() {
.expect(&format!("Could not launch tldr binary ({:?})", &testenv.bin_path));
assert_eq!(out3.status.success(), true);
}
fn _test_correct_rendering(filename: &str) {
let testenv = TestEnv::new();
let testfile = testenv.tests_path.join(filename);
let testfile_expected = testenv.tests_path.join("inkscape.expected");
let out: Vec<u8> = testenv.cmd()
.arg("-f").arg(testfile)
.output()
.expect(&format!("Could not launch tldr binary ({:?})", &testenv.bin_path))
.stdout;
let mut expected = Vec::<u8>::new();
BufReader::new(File::open(testfile_expected).unwrap()).read_to_end(&mut expected).unwrap();
assert_eq!(out, expected);
}
/// An end-to-end integration test for direct file rendering.
#[test]
fn test_correct_rendering_v1() {
_test_correct_rendering("inkscape-v1.md");
}
/// An end-to-end integration test for direct file rendering.
#[test]
fn test_correct_rendering_v2() {
_test_correct_rendering("inkscape-v2.md");
}