From ab44af7d36d32d50c77f0a4297334a0fd3e5e82f Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Sun, 3 Jan 2016 23:41:52 +0100 Subject: [PATCH] Deserialize index.json, copy tldr pages to cache --- Cargo.lock | 1 + Cargo.toml | 1 + src/main.rs | 4 +- src/updater.rs | 99 +++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f21eab0..adce977 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7,6 +7,7 @@ dependencies = [ "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.3.2 (git+https://github.com/dbrgn/tar-rs?branch=pax_header)", "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/Cargo.toml b/Cargo.toml index 9c8b3b5..513d69d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,3 +14,4 @@ flate2 = "^0.2" tempdir = "^0.3" curl = "^0.2" env_logger = { version = "^0.3", optional = true } +rustc-serialize = "^0.3" diff --git a/src/main.rs b/src/main.rs index 24895dd..403eca5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ extern crate flate2; extern crate tar; extern crate tempdir; extern crate curl; +extern crate rustc_serialize; use std::io::{BufRead, BufReader}; use std::fs::File; @@ -85,12 +86,13 @@ fn main() { println!(""); let dl = Updater::new("https://github.com/tldr-pages/tldr/archive/master.tar.gz".into()); - dl.update().unwrap_or_else(|e| { + let copied = dl.update().unwrap_or_else(|e| { match e { TldrError::UpdateError(msg) => println!("Could not update cache: {}", msg), }; process::exit(1); }); + println!("Cached {} tldr pages.", copied); } diff --git a/src/updater.rs b/src/updater.rs index 8835d23..8d4760a 100644 --- a/src/updater.rs +++ b/src/updater.rs @@ -1,5 +1,4 @@ use std::io::Read; -use std::io::Result as IoResult; use std::path::{Path, PathBuf}; use std::fs; use std::env; @@ -8,10 +7,35 @@ use flate2::read::GzDecoder; use tar::Archive; use curl::http; use tempdir::TempDir; +use rustc_serialize::json; use error::TldrError::{self, UpdateError}; +#[derive(Debug, RustcEncodable, RustcDecodable)] +struct TldrIndex { + commands: Vec, +} + +#[derive(Debug, RustcEncodable, RustcDecodable)] +struct TldrCommand { + name: String, + platform: Vec, +} + +impl TldrIndex { + /// Return vec of all occuring platforms, without duplicates. + fn unique_platforms(&self) -> Vec { + let mut flattened: Vec = self.commands + .iter() + .flat_map(|cmd| cmd.platform.clone()) + .collect(); + flattened.sort(); + flattened.dedup(); + flattened + } +} + #[derive(Debug)] pub struct Updater { url: String, @@ -51,11 +75,50 @@ impl Updater { Ok((repodir.join("pages"), repodir.join("LICENSE.md"))) } - fn get_pages(&self, path: &Path) -> Vec { + /// Given the path to the `pages` directory, return `TldrIndex` instances. + fn get_index(&self, path: &Path) -> Result { + let mut buffer = String::new(); + let mut index = try!(fs::File::open(path.join("index.json")) + .map_err(|_| UpdateError("Could not open index.json".into()))); + try!(index.read_to_string(&mut buffer) + .map_err(|_| UpdateError("Could not read index.json".into()))); + let deserialized: TldrIndex = try!(json::decode(&buffer) + .map_err(|e| UpdateError(format!("Could not deserialize index.json: {}", e)))); + Ok(deserialized) } - /// Update the pages cache. - pub fn update(&self) -> Result<(), TldrError> { + /// Copy all pages to the cache. Return the number of copied pages. + fn copy_pages(&self, src_dir: &Path, dst_dir: &Path, index: &TldrIndex) -> Result { + // Make sure all platform directories exist + for platform in &index.unique_platforms() { + debug!("Ensure platform directory {:?} exists", &dst_dir.join(platform)); + try!(fs::create_dir_all(&dst_dir.join(platform)).map_err(|e| { + UpdateError(format!("Could not create platform directory: {}", e)) + })); + } + + let mut copied = 0u64; + for page in &index.commands { + for platform in &page.platform { + let relpath = Path::new(&platform).join(format!("{}.md", &page.name)); + let bytes = fs::copy(&src_dir.join(&relpath), &dst_dir.join(&relpath)).unwrap_or_else(|e| { + debug!("Could not copy {:?} to {:?}: {}", + &src_dir.join(&relpath), &dst_dir.join(&relpath), e); + println!("Warning: Could not copy the tldr page for the \"{}\" command.", &page.name); + 0u64 + }); + if bytes > 0 { + copied += 1; + }; + } + }; + + debug!("Copied {} pages to {:?}", copied, &dst_dir); + Ok(copied) + } + + /// Update the pages cache. Return the number of cached pages. + pub fn update(&self) -> Result { // First, download the compressed data let response = try!(self.download()); @@ -67,26 +130,44 @@ impl Updater { UpdateError(format!("Could not create temporary directory: {}", e)) })); - // Get paths to pages and license + // Extract archive and get paths to pages and license let (pages_src, license_src) = try!(self.extract(&mut archive, dir.path())); - // Determine path to cache + // Determine paths let home_dir = try!(env::home_dir().ok_or(UpdateError("Could not determine home directory".into()))); let cache_dir = home_dir.join(".tldr").join("cache"); + let pages_dir = &cache_dir.join("pages"); + let license_dst = &cache_dir.join("LICENSE.md"); + let index_dst = &pages_dir.join("index.json"); - // Make sure that cache dir exists + // Make sure that cache and pages directories exist + debug!("Ensure cache directory {:?} exists", &cache_dir); try!(fs::create_dir_all(&cache_dir).map_err(|e| { UpdateError(format!("Could not create cache directory: {}", e)) })); + debug!("Ensure pages directory {:?} exists", &pages_dir); + try!(fs::create_dir_all(&pages_dir).map_err(|e| { + UpdateError(format!("Could not create pages directory: {}", e)) + })); // Copy license file - let license_dst = &cache_dir.join("LICENSE.md"); debug!("Copy license from {:?} to {:?}", &license_src, &license_dst); try!(fs::copy(&license_src, &license_dst).map_err(|e| { UpdateError(format!("Could not extract license file: {}", e)) })); + + // Copy index file + let index_src = &pages_src.join("index.json"); + debug!("Copy index from {:?} to {:?}", &index_src, &index_dst); + try!(fs::copy(&index_src, &index_dst).map_err(|e| { + UpdateError(format!("Could not extract index file: {}", e)) + })); + + // Copy pages + let index = try!(self.get_index(&pages_src)); + let copied = try!(self.copy_pages(&pages_src, &pages_dir, &index)); - Ok(()) + Ok(copied) } }