chroot_builder.rs

use super::DocBuilder;
use super::crates::crates_from_path;
use super::metadata::Metadata;
use utils::{get_package, source_path, copy_doc_dir,
            update_sources, parse_rustc_version, command_result};
use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database};
use cargo::core::Package;
use cargo::util::CargoResultExt;
use std::process::Command;
use std::path::PathBuf;
use std::fs::remove_dir_all;
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use error::Result;


/// List of targets supported by docs.rs
const TARGETS: [&'static str; 6] = [
    "i686-apple-darwin",
    "i686-pc-windows-msvc",
    "i686-unknown-linux-gnu",
    "x86_64-apple-darwin",
    "x86_64-pc-windows-msvc",
    "x86_64-unknown-linux-gnu"
];



#[derive(Debug)]
pub struct ChrootBuilderResult {
    pub output: String,
    pub build_success: bool,
    pub have_doc: bool,
    pub have_examples: bool,
    pub rustc_version: String,
    pub cratesfyi_version: String,
}


impl DocBuilder {
    /// Builds every package documentation in chroot environment
    pub fn build_world(&mut self) -> Result<()> {
        try!(update_sources());

        let mut count = 0;

        crates(self.options.crates_io_index_path.clone(), |name, version| {
            match self.build_package(name, version) {
                Ok(status) => {
                    count += 1;
                    if status && count % 10 == 0 {
                        let _ = self.save_cache();
                    }
                }
                Err(err) => warn!("Failed to build package {}-{}: {}", name, version, err),
            }
            self.cache.insert(format!("{}-{}", name, version));
        })
    }


    /// Builds package documentation in chroot environment and adds into cratesfyi database
    pub fn build_package(&mut self, name: &str, version: &str) -> Result<bool> {
        // Skip crates according to options
        if (self.options.skip_if_log_exists &&
            self.cache.contains(&format!("{}-{}", name, version)[..])) ||
           (self.options.skip_if_exists &&
            self.db_cache.contains(&format!("{}-{}", name, version)[..])) {
            return Ok(false);
        }

        info!("Building package {}-{}", name, version);

        // Start with clean documentation directory
        try!(self.remove_build_dir());

        // Database connection
        let conn = try!(connect_db());

        // get_package (and cargo) is using semver, add '=' in front of version.
        let pkg = try!(get_package(name, Some(&format!("={}", version)[..])));
        let metadata = Metadata::from_package(&pkg)?;
        let res = self.build_package_in_chroot(&pkg, metadata.default_target.clone());

        // copy sources and documentation
        let file_list = try!(self.add_sources_into_database(&conn, &pkg));
        let successfully_targets = if res.have_doc {
            try!(self.copy_documentation(&pkg,
                                         &res.rustc_version,
                                         metadata.default_target.as_ref().map(String::as_str),
                                         true));
            let successfully_targets = self.build_package_for_all_targets(&pkg);
            for target in &successfully_targets {
                try!(self.copy_documentation(&pkg, &res.rustc_version, Some(target), false));
            }
            try!(self.add_documentation_into_database(&conn, &pkg));
            successfully_targets
        } else {
            Vec::new()
        };

        let release_id = try!(add_package_into_database(&conn,
                                                        &pkg,
                                                        &res,
                                                        Some(file_list),
                                                        successfully_targets));
        try!(add_build_into_database(&conn, &release_id, &res));

        // remove documentation, source and build directory after we are done
        try!(self.clean(&pkg));

        // add package into build cache
        self.cache.insert(format!("{}-{}", name, version));

        Ok(res.build_success)
    }


    /// Builds documentation of a package with cratesfyi in chroot environment
    fn build_package_in_chroot(&self, package: &Package, default_target: Option<String>) -> ChrootBuilderResult {
        debug!("Building package in chroot");
        let (rustc_version, cratesfyi_version) = self.get_versions();
        let cmd = format!("cratesfyi doc {} ={} {}",
                          package.manifest().name(),
                          package.manifest().version(),
                          default_target.as_ref().unwrap_or(&"".to_string()));
        match self.chroot_command(cmd) {
            Ok(o) => {
                ChrootBuilderResult {
                    output: o,
                    build_success: true,
                    have_doc: self.have_documentation(&package, default_target),
                    have_examples: self.have_examples(&package),
                    rustc_version: rustc_version,
                    cratesfyi_version: cratesfyi_version,
                }
            }
            Err(e) => {
                ChrootBuilderResult {
                    output: e.to_string(),
                    build_success: false,
                    have_doc: false,
                    have_examples: self.have_examples(&package),
                    rustc_version: rustc_version,
                    cratesfyi_version: cratesfyi_version,
                }
            }
        }
    }



    /// Builds documentation of crate for every target and returns Vec of successfully targets
    fn build_package_for_all_targets(&self, package: &Package) -> Vec<String> {
        let mut successfuly_targets = Vec::new();

        for target in TARGETS.iter() {
            debug!("Building {} for {}", canonical_name(&package), target);
            let cmd = format!("cratesfyi doc {} ={} {}",
                              package.manifest().name(),
                              package.manifest().version(),
                              target);
            if let Ok(_) = self.chroot_command(cmd) {
                // Cargo is not giving any error and not generating documentation of some crates
                // when we use a target compile options. Check documentation exists before
                // adding target to successfully_targets.
                // FIXME: Need to figure out why some docs are not generated with target option
                let target_doc_path = PathBuf::from(&self.options.chroot_path)
                    .join("home")
                    .join(&self.options.chroot_user)
                    .join("cratesfyi")
                    .join(&target)
                    .join("doc");
                if target_doc_path.exists() {
                    successfuly_targets.push(target.to_string());
                }
            }
        }
        successfuly_targets
    }


    /// Copies documentation to destination directory
    fn copy_documentation(&self,
                          package: &Package,
                          rustc_version: &str,
                          target: Option<&str>,
                          is_default_target: bool)
                          -> Result<()> {
        let mut crate_doc_path = PathBuf::from(&self.options.chroot_path)
            .join("home")
            .join(&self.options.chroot_user)
            .join("cratesfyi");

        // docs are available in cratesfyi/$TARGET when target is being used
        if let Some(target) = target {
            crate_doc_path.push(target);
        }

        let mut destination = PathBuf::from(&self.options.destination)
            .join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));

        // only add target name to destination directory when we are copying a non-default target.
        // this is allowing us to host documents in the root of the crate documentation directory.
        // for example winapi will be available in docs.rs/winapi/$version/winapi/ for it's
        // default target: x86_64-pc-windows-msvc. But since it will be built under
        // cratesfyi/x86_64-pc-windows-msvc we still need target in this function.
        if !is_default_target {
            if let Some(target) = target {
                destination.push(target);
            }
        }

        copy_doc_dir(crate_doc_path,
                     destination,
                     parse_rustc_version(rustc_version)?.trim())
    }


    /// Removes build directory of a package in chroot
    fn remove_build_dir(&self) -> Result<()> {
        let crate_doc_path = PathBuf::from(&self.options.chroot_path)
            .join("home")
            .join(&self.options.chroot_user)
            .join("cratesfyi")
            .join("doc");
        let _ = remove_dir_all(crate_doc_path);
        for target in TARGETS.iter() {
            let crate_doc_path = PathBuf::from(&self.options.chroot_path)
                .join("home")
                .join(&self.options.chroot_user)
                .join("cratesfyi")
                .join(target)
                .join("doc");
            let _ = remove_dir_all(crate_doc_path);
        }
        Ok(())
    }


    /// Remove documentation, build directory and sources directory of a package
    fn clean(&self, package: &Package) -> Result<()> {
        debug!("Cleaning package");
        use std::fs::remove_dir_all;
        let documentation_path = PathBuf::from(&self.options.destination)
            .join(package.manifest().name().as_str());
        let source_path = source_path(&package).unwrap();
        // Some crates don't have documentation, so we don't care if removing_dir_all fails
        let _ = self.remove_build_dir();
        let _ = remove_dir_all(documentation_path);
        let _ = remove_dir_all(source_path);
        Ok(())
    }


    /// Runs a command in a chroot environment
    fn chroot_command<T: AsRef<str>>(&self, cmd: T) -> Result<String> {
        command_result(Command::new("sudo")
            .arg("lxc-attach")
            .arg("-n")
            .arg(&self.options.container_name)
            .arg("--")
            .arg("su")
            .arg("-")
            .arg(&self.options.chroot_user)
            .arg("-c")
            .arg(cmd.as_ref())
            .output()
            .unwrap())
    }


    /// Checks a package build directory to determine if package have docs
    ///
    /// This function is checking first target in targets to see if documentation exists for a
    /// crate. Package must be successfully built in chroot environment first.
    fn have_documentation(&self, package: &Package, default_target: Option<String>) -> bool {
        let mut crate_doc_path = PathBuf::from(&self.options.chroot_path)
            .join("home")
            .join(&self.options.chroot_user)
            .join("cratesfyi");

        if let Some(default_doc_path) = default_target {
            crate_doc_path.push(default_doc_path);
        }

        crate_doc_path.push("doc");
        crate_doc_path.push(package.targets()[0].name().replace("-", "_").to_string());
        crate_doc_path.exists()
    }


    /// Checks if package have examples
    fn have_examples(&self, package: &Package) -> bool {
        let path = source_path(&package).unwrap().join("examples");
        path.exists() && path.is_dir()
    }


    /// Gets rustc and cratesfyi version from chroot environment
    pub fn get_versions(&self) -> (String, String) {
        // It is safe to use expect here
        // chroot environment must always have rustc and cratesfyi installed
        (String::from(self.chroot_command("rustc --version")
            .expect("Failed to get rustc version")
            .trim()),
         String::from(self.chroot_command("cratesfyi --version")
            .expect("Failed to get cratesfyi version")
            .trim()))
    }


    /// Adds sources into database
    fn add_sources_into_database(&self, conn: &Connection, package: &Package) -> Result<Json> {
        debug!("Adding sources into database");
        let prefix = format!("sources/{}/{}",
                             package.manifest().name(),
                             package.manifest().version());
        add_path_into_database(conn, &prefix, source_path(&package).unwrap())
    }


    /// Adds documentations into database
    fn add_documentation_into_database(&self,
                                       conn: &Connection,
                                       package: &Package)
                                       -> Result<Json> {
        debug!("Adding documentation into database");
        let prefix = format!("rustdoc/{}/{}",
                             package.manifest().name(),
                             package.manifest().version());
        let crate_doc_path = PathBuf::from(&self.options.destination).join(format!("{}/{}",
                          package.manifest().name(),
                          package.manifest().version()));
        add_path_into_database(conn, &prefix, crate_doc_path)
    }


    /// This function will build an empty crate and will add essential documentation files.
    ///
    /// It is required to run after every rustc update. cratesfyi is not keeping this files
    /// for every crate to avoid duplications.
    ///
    /// List of the files:
    ///
    /// * rustdoc.css (with rustc version)
    /// * main.css (with rustc version)
    /// * main.js (with rustc version)
    /// * jquery.js (with rustc version)
    /// * playpen.js (with rustc version)
    /// * normalize.css
    /// * FiraSans-Medium.woff
    /// * FiraSans-Regular.woff
    /// * Heuristica-Italic.woff
    /// * SourceCodePro-Regular.woff
    /// * SourceCodePro-Semibold.woff
    /// * SourceSerifPro-Bold.woff
    /// * SourceSerifPro-Regular.woff
    pub fn add_essential_files(&self) -> Result<()> {
        use std::fs::{copy, create_dir_all};

        // acme-client-0.0.0 is an empty library crate and it will always build
        let pkg = try!(get_package("acme-client", Some("=0.0.0")));
        let res = self.build_package_in_chroot(&pkg, None);
        let rustc_version = parse_rustc_version(&res.rustc_version)?;

        if !res.build_success {
            return Err(format_err!("Failed to build empty crate for: {}", res.rustc_version));
        }

        info!("Copying essential files for: {}", res.rustc_version);

        let files = (// files require rustc version subfix
                     ["brush.svg",
                      "wheel.svg",
                      "down-arrow.svg",
                      "dark.css",
                      "light.css",
                      "main.js",
                      "normalize.css",
                      "rustdoc.css",
                      "settings.css",
                      "storage.js",
                      "theme.js",
                      "source-script.js",
                      "noscript.css"],
                     // files doesn't require rustc version subfix
                     ["FiraSans-Medium.woff",
                      "FiraSans-Regular.woff",
                      "Heuristica-Italic.woff",
                      "SourceCodePro-Regular.woff",
                      "SourceCodePro-Semibold.woff",
                      "SourceSerifPro-Bold.woff",
                      "SourceSerifPro-Regular.woff"]);

        let source = PathBuf::from(&self.options.chroot_path)
            .join("home")
            .join(&self.options.chroot_user)
            .join("cratesfyi")
            .join("doc");

        // use copy_documentation destination directory so self.clean can remove it when
        // we are done
        let destination = PathBuf::from(&self.options.destination)
            .join(format!("{}/{}", pkg.manifest().name(), pkg.manifest().version()));
        try!(create_dir_all(&destination));

        for file in files.0.iter() {
            let spl: Vec<&str> = file.split('.').collect();
            let file_name = format!("{}-{}.{}", spl[0], rustc_version, spl[1]);
            let source_path = source.join(&file_name);
            let destination_path = destination.join(&file_name);
            try!(copy(&source_path, &destination_path)
                .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display())));
        }

        for file in files.1.iter() {
            let source_path = source.join(file);
            let destination_path = destination.join(file);
            try!(copy(&source_path, &destination_path)
                .chain_err(|| format!("couldn't copy '{}' to '{}'", source_path.display(), destination_path.display())));
        }

        let conn = try!(connect_db());
        try!(add_path_into_database(&conn, "", destination));

        try!(self.clean(&pkg));

        let (vers, _) = self.get_versions();

        try!(conn.query("INSERT INTO config (name, value) VALUES ('rustc_version', $1)",
                   &[&vers.to_json()])
            .or_else(|_| {
                conn.query("UPDATE config SET value = $1 WHERE name = 'rustc_version'",
                           &[&vers.to_json()])
            }));

        Ok(())
    }
}


/// Returns canonical name of a package.
///
/// It's just package-version. All directory structure used in cratesfyi is
/// following this naming scheme.
fn canonical_name(package: &Package) -> String {
    format!("{}-{}",
            package.manifest().name(),
            package.manifest().version())
}


/// Runs `func` with the all crates from crates-io.index repository path.
///
/// First argument of func is the name of crate and
/// second argument is the version of crate. Func will be run for every crate.
fn crates<F>(path: PathBuf, mut func: F) -> Result<()>
    where F: FnMut(&str, &str) -> ()
{
    crates_from_path(&path, &mut func)
}


#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::path::PathBuf;
    use {DocBuilder, DocBuilderOptions};

    #[test]
    #[ignore]
    fn test_build_world() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        // This test is building WHOLE WORLD and may take forever
        assert!(docbuilder.build_world().is_ok());
    }

    #[test]
    #[ignore]
    fn test_build_package() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let mut docbuilder = DocBuilder::new(options);
        let res = docbuilder.build_package("rand", "0.3.14");
        assert!(res.is_ok());
    }

    #[test]
    #[ignore]
    fn test_add_essential_files() {
        let _ = env_logger::try_init();
        let options = DocBuilderOptions::from_prefix(PathBuf::from("../cratesfyi-prefix"));
        let docbuilder = DocBuilder::new(options);

        docbuilder.add_essential_files().unwrap();
    }
}