From e7e3211cd5dd7d523e6ef0fe4b04e65a6997ef5a Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Fri, 1 Jan 2016 15:19:52 +0100 Subject: [PATCH] Download and decompress tldr pages --- Cargo.lock | 210 +++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 4 + src/error.rs | 14 ++++ src/main.rs | 16 ++++ src/updater.rs | 83 +++++++++++++++++++ 5 files changed, 327 insertions(+) create mode 100644 src/error.rs create mode 100644 src/updater.rs diff --git a/Cargo.lock b/Cargo.lock index 1e5b92e..f21eab0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3,8 +3,21 @@ name = "tldr" version = "0.1.0" dependencies = [ "ansi_term 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "curl 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.3.2 (git+https://github.com/dbrgn/tar-rs?branch=pax_header)", + "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "advapi32-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -20,6 +33,30 @@ name = "ansi_term" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "curl" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "curl-sys 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "curl-sys" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.21 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libz-sys 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "env_logger" version = "0.3.2" @@ -29,11 +66,74 @@ dependencies = [ "regex 0.1.44 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "filetime" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "flate2" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz-sys 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gcc" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "advapi32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gdi32-sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libc" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "libressl-pnacl-sys" +version = "2.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libz-sys" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.21 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "log" version = "0.3.4" @@ -42,6 +142,11 @@ dependencies = [ "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "matches" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "memchr" version = "0.1.7" @@ -50,6 +155,50 @@ dependencies = [ "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "miniz-sys" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.21 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-sys" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gdi32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "user32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pkg-config" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pnacl-build-helper" +version = "1.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "advapi32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "0.1.44" @@ -65,3 +214,64 @@ name = "regex-syntax" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "rustc-serialize" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tar" +version = "0.3.2" +source = "git+https://github.com/dbrgn/tar-rs?branch=pax_header#b573100243c34347fbeba6345bfbba5a47b16dd0" +dependencies = [ + "filetime 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tempdir" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "user32-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uuid" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + diff --git a/Cargo.toml b/Cargo.toml index bd61a84..9c8b3b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,4 +9,8 @@ logging = ["env_logger"] [dependencies] ansi_term = "^0.7" log = "^0.3" +tar = { git = "https://github.com/dbrgn/tar-rs", branch = "pax_header" } +flate2 = "^0.2" +tempdir = "^0.3" +curl = "^0.2" env_logger = { version = "^0.3", optional = true } diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..6af0965 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,14 @@ +use curl::ErrCode; + + +#[derive(Debug)] +pub enum TldrError { + UpdateError(String) +} + + +impl From for TldrError { + fn from(err: ErrCode) -> TldrError { + TldrError::UpdateError(err.to_string()) + } +} diff --git a/src/main.rs b/src/main.rs index c1fd06a..24895dd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,10 @@ #[macro_use] extern crate log; #[cfg(feature = "logging")]extern crate env_logger; extern crate ansi_term; +extern crate flate2; +extern crate tar; +extern crate tempdir; +extern crate curl; use std::io::{BufRead, BufReader}; use std::fs::File; @@ -12,9 +16,13 @@ use ansi_term::Colour; mod types; mod tokenizer; +mod updater; +mod error; use types::LineType; use tokenizer::Tokenizer; +use updater::Updater; +use error::TldrError; /// Open file, return a `BufRead` instance @@ -76,6 +84,14 @@ fn main() { print_lines(&mut tokenizer); println!(""); + let dl = Updater::new("https://github.com/tldr-pages/tldr/archive/master.tar.gz".into()); + dl.update().unwrap_or_else(|e| { + match e { + TldrError::UpdateError(msg) => println!("Could not update cache: {}", msg), + }; + process::exit(1); + }); + } diff --git a/src/updater.rs b/src/updater.rs new file mode 100644 index 0000000..345c615 --- /dev/null +++ b/src/updater.rs @@ -0,0 +1,83 @@ +use std::io::Read; +use std::io::Result as IoResult; +use std::path::{Path, PathBuf}; +use std::fs; +use std::env; + +use flate2::read::GzDecoder; +use tar::Archive; +use curl::http; +use tempdir::TempDir; + +use error::TldrError::{self, UpdateError}; + + +#[derive(Debug)] +pub struct Updater { + url: String, +} + +impl Updater { + + pub fn new(url: String) -> Updater { + Updater { + url: url, + } + } + + /// Download the archive + fn download(&self) -> Result { + let resp = try!( + http::handle() + .follow_location(1) + .get(&self.url[..]) + .exec() + ); + Ok(resp) + } + + /// Decompress and open the archive + fn decompress(&self, reader: R) -> Result>, TldrError> { + let decoder = try!(GzDecoder::new(reader).map_err(|_| UpdateError("Could not decode gzip data".into()))); + Ok(Archive::new(decoder)) + } + + /// Extract archive, return pages and license + fn extract(&self, archive: &mut Archive, path: &Path) -> Result<(PathBuf, PathBuf), TldrError> { + try!(archive.unpack(path).map_err(|e| { + UpdateError(format!("Could not unpack compressed data: {}", e)) + })); + let repodir = path.join("tldr-master"); + Ok((repodir.join("pages"), repodir.join("LICENSE.md"))) + } + + fn get_pages(&self, path: &Path) -> Vec { + } + + /// Update the pages cache. + pub fn update(&self) -> Result<(), TldrError> { + // First, download the compressed data + let response = try!(self.download()); + + // Decompress the response body into an `Archive` + let mut archive = try!(self.decompress(response.get_body())); + + // Create temporary directory + let dir = try!(TempDir::new("tldr").map_err(|e| { + UpdateError(format!("Could not create temporary directory: {}", e)) + })); + + // Get paths to pages and license + let (pages_path, license_path) = try!(self.extract(&mut archive, dir.path())); + + // Copy files + let home_dir = try!(env::home_dir().ok_or(UpdateError("Could not determine home directory".into()))); + let cache_dir = home_dir.join(".tldr").join("cache"); + try!(fs::copy(license_path, cache_dir.join("LICENSE.md")).map_err(|e| { + UpdateError(format!("Could not extract license file: {}", e)) + })); + + Ok(()) + } + +}