Skip to content

Commit

Permalink
Support for running wasm-opt, commented-out stubs for production prof…
Browse files Browse the repository at this point in the history 
…ile.
  • Loading branch information
@mwu-tow
mwu-tow committed Jun 10, 2022
1 parent 5f4d657 commit 2fbb1bc
Show file tree
Hide file tree
Showing 21 changed files with 398 additions and 93 deletions.
5 changes: 4 additions & 1 deletion build/src/build2.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,10 @@ pub trait BuildContext: Clone + Send + Sync + 'static {
async move {
let ReleaseSource { asset_id, octocrab, repository } = &source;
let archive_source = repository.download_asset_job(octocrab, *asset_id);
let extract_job = cache::archive::ExtractedArchive { archive_source };
let extract_job = cache::archive::ExtractedArchive {
archive_source,
path_to_extract: crate::project::path_to_extract(),
};
let directory = cache.get(extract_job).await?;
ide_ci::fs::remove_if_exists(&destination)?;
ide_ci::fs::symlink_auto(&directory, &destination)?;
Expand Down
6 changes: 4 additions & 2 deletions build/src/enso.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,10 @@ impl Program for BuiltEnso {
ide_ci::platform::DEFAULT_SHELL.run_script(self.wrapper_script_path())
}

async fn version_string(&self) -> Result<String> {
self.cmd()?.args(["version", "--json", "--only-launcher"]).run_stdout().await
fn version_string(&self) -> BoxFuture<'static, Result<String>> {
let command = self.cmd();
async move { command?.args(["version", "--json", "--only-launcher"]).run_stdout().await }
.boxed()
}

async fn version(&self) -> Result<Version> {
Expand Down
10 changes: 9 additions & 1 deletion build/src/project.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ pub use ide::Ide;
pub use project_manager::ProjectManager;
pub use wasm::Wasm;

// FIXME: this works for Project Manager bundle-style archives only, not all.
pub fn path_to_extract() -> Option<PathBuf> {
Some("enso".into())
}

/// A built target, contained under a single directory.
///
/// The `AsRef<Path>` trait must return that directory path.
Expand Down Expand Up @@ -184,7 +189,10 @@ pub trait IsTarget: Clone + Debug + Sized + Send + Sync + 'static {
async move {
let ReleaseSource { asset_id, octocrab, repository } = &source;
let archive_source = repository.download_asset_job(octocrab, *asset_id);
let extract_job = cache::archive::ExtractedArchive { archive_source };
let extract_job = cache::archive::ExtractedArchive {
archive_source,
path_to_extract: path_to_extract(),
};
let directory = cache.get(extract_job).await?;
ide_ci::fs::remove_if_exists(&destination)?;
ide_ci::fs::symlink_auto(&directory, &destination)?;
Expand Down
92 changes: 83 additions & 9 deletions build/src/project/wasm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,13 @@ use crate::project::IsWatchable;

use anyhow::Context;
use derivative::Derivative;
use ide_ci::cache;
use ide_ci::cache::Cache;
use ide_ci::env::Variable;
use ide_ci::fs::compressed_size;
use ide_ci::programs::cargo;
use ide_ci::programs::wasm_opt;
use ide_ci::programs::wasm_opt::WasmOpt;
use ide_ci::programs::wasm_pack;
use ide_ci::programs::Cargo;
use ide_ci::programs::WasmPack;
Expand All @@ -26,6 +30,8 @@ pub mod env;
pub mod js_patcher;
pub mod test;

pub const BINARYEN_VERSION_TO_INSTALL: usize = 108;

pub const DEFAULT_INTEGRATION_TESTS_WASM_TIMEOUT: Duration = Duration::from_secs(300);

pub const INTEGRATION_TESTS_CRATE_NAME: &str = "enso-integration-test";
Expand All @@ -50,6 +56,58 @@ pub enum ProfilingLevel {
Debug,
}

#[derive(clap::ArgEnum, Clone, Copy, Debug, PartialEq, strum::Display, strum::AsRefStr)]
#[strum(serialize_all = "kebab-case")]
pub enum Profile {
Dev,
Profile,
Release,
// Production,
}

impl From<Profile> for wasm_pack::Profile {
fn from(profile: Profile) -> Self {
match profile {
Profile::Dev => Self::Dev,
Profile::Profile => Self::Profile,
Profile::Release => Self::Release,
// Profile::Production => Self::Release,
}
}
}

impl Profile {
pub fn should_check_size(self) -> bool {
match self {
Profile::Dev => false,
Profile::Profile => false,
Profile::Release => true,
// Profile::Production => true,
}
}

pub fn extra_rust_options(self) -> Vec<String> {
match self {
// Profile::Production => ["-Clto=fat", "-Ccodegen-units=1", "-Cincremental=false"]
// .into_iter()
// .map(ToString::to_string)
// .collect(),
Profile::Dev | Profile::Profile | Profile::Release => vec![],
}
}

/// wasm-opt invocation that should follow the wasm-pack build for this profile.
pub fn wasm_opt_command(self) -> Result<Option<Command>> {
let opt_level = match self {
Profile::Dev => return Ok(None),
Profile::Profile => wasm_opt::OptimizationLevel::O,
Profile::Release => wasm_opt::OptimizationLevel::O3,
// Profile::Production => wasm_opt::OptimizationLevel::O4,
};
Ok(Some(WasmOpt.cmd()?.with_applied(&opt_level)))
}
}

#[derive(Clone, Derivative)]
#[derivative(Debug)]
pub struct BuildInput {
Expand All @@ -58,9 +116,10 @@ pub struct BuildInput {
/// Path to the crate to be compiled to WAM. Relative to the repository root.
pub crate_path: PathBuf,
pub extra_cargo_options: Vec<String>,
pub profile: wasm_pack::Profile,
pub profile: Profile,
pub profiling_level: Option<ProfilingLevel>,
pub wasm_size_limit: Option<byte_unit::Byte>,
pub cache: Cache,
}

impl BuildInput {
Expand All @@ -69,12 +128,8 @@ impl BuildInput {
info!("Compressed size of {} is {}.", wasm_path.as_ref().display(), compressed_size);
if let Some(wasm_size_limit) = self.wasm_size_limit {
let wasm_size_limit = wasm_size_limit.get_appropriate_unit(true);
if self.profile != wasm_pack::Profile::Release {
warn!(
"Skipping size check because profile is {} rather than {}.",
self.profile,
wasm_pack::Profile::Release
);
if !self.profile.should_check_size() {
warn!("Skipping size check because profile is {}.", self.profile,);
} else if self.profiling_level.unwrap_or_default() != ProfilingLevel::Objective {
// TODO? additional leeway as sanity check
warn!(
Expand Down Expand Up @@ -126,15 +181,23 @@ impl IsTarget for Wasm {
// We want to be able to pass --profile this way.
WasmPack.require_present_that(&VersionReq::parse(">=0.10.1")?).await?;


let BuildInput {
repo_root,
crate_path,
extra_cargo_options,
profile,
profiling_level,
wasm_size_limit: _wasm_size_limit,
cache,
} = &input;


cache::goodie::binaryen::Binaryen { version: BINARYEN_VERSION_TO_INSTALL }
.install_if_missing(&cache, WasmOpt)
.await?;


info!("Building wasm.");
let temp_dir = tempdir()?;
let temp_dist = RepoRootDistWasm::new(temp_dir.path());
Expand All @@ -145,7 +208,7 @@ impl IsTarget for Wasm {
.env_remove(ide_ci::programs::rustup::env::Toolchain::NAME)
.set_env(env::ENSO_ENABLE_PROC_MACRO_SPAN, &true)?
.build()
.arg(&profile)
.arg(&wasm_pack::Profile::from(*profile))
.target(wasm_pack::Target::Web)
.output_directory(&temp_dist)
.output_name(&OUTPUT_NAME)
Expand All @@ -159,8 +222,18 @@ impl IsTarget for Wasm {
}
command.run_ok().await?;

if let Some(mut wasm_opt_command) = profile.wasm_opt_command()? {
wasm_opt_command
.arg(&temp_dist.wasm_main_raw)
.apply(&wasm_opt::Output(&temp_dist.wasm_main))
.run_ok()
.await?;
} else {
debug!("Skipping wasm-opt invocation, as it is not part of profile {profile}.")
}

// ide_ci::fs::rename(&temp_dist.wasm_main_raw, &temp_dist.wasm_main)?;
patch_js_glue_in_place(&temp_dist.wasm_glue)?;
ide_ci::fs::rename(&temp_dist.wasm_main_raw, &temp_dist.wasm_main)?;

ide_ci::fs::create_dir_if_missing(&output_path)?;
let ret = RepoRootDistWasm::new(output_path.as_ref());
Expand Down Expand Up @@ -194,6 +267,7 @@ impl IsWatchable for Wasm {
profile,
profiling_level,
wasm_size_limit,
cache: _,
} = input;

let current_exe = std::env::current_exe()?;
Expand Down
28 changes: 28 additions & 0 deletions ci_utils/src/anyhow.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,21 @@ use anyhow::Error;

pub trait ResultExt<T, E> {
fn anyhow_err(self) -> Result<T>;

fn flatten_fut(
self,
) -> futures::future::Either<
std::future::Ready<std::result::Result<T::Ok, T::Error>>,
futures::future::IntoFuture<T>,
>
where T: TryFuture<Error: From<E>>;

// fn flatten_fut(self) -> impl Future<Output = std::result::Result<T::Ok, T::Error>>
// where T: TryFuture<Error: From<E>> {
// async move { fut?.into_future().await }
// }
// fn flatten_fut(self)
// where T: TryFuture;
}

impl<T, E> ResultExt<T, E> for std::result::Result<T, E>
Expand All @@ -11,4 +26,17 @@ where E: Into<Error>
fn anyhow_err(self) -> Result<T> {
self.map_err(E::into)
}

fn flatten_fut(
self,
) -> futures::future::Either<
std::future::Ready<std::result::Result<T::Ok, T::Error>>,
futures::future::IntoFuture<T>,
>
where T: TryFuture<Error: From<E>> {
match self {
Ok(fut) => fut.into_future().right_future(),
Err(e) => ready(std::result::Result::Err(T::Error::from(e))).left_future(),
}
}
}
49 changes: 29 additions & 20 deletions ci_utils/src/cache.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ pub mod archive;
pub mod artifact;
pub mod asset;
pub mod download;
pub mod goodie;

use crate::prelude::*;
use anyhow::Context;
Expand All @@ -10,13 +11,15 @@ use std::hash::Hasher;
use serde::de::DeserializeOwned;
use sha2::Digest;

pub use goodie::Goodie;

pub const VERSION: u8 = 1;

pub fn default_path() -> Result<PathBuf> {
Ok(dirs::home_dir().context("Cannot locate home directory.")?.join_iter([".enso-ci", "cache"]))
}

pub trait Storable: Debug + Borrow<Self::Key> + Send + Sync + 'static {
pub trait Storable: Debug + Send + Sync + 'static {
type Metadata: Serialize + DeserializeOwned + Send + Sync + 'static;
type Output: Clone + Send + Sync + 'static;
type Key: Clone + Debug + Serialize + DeserializeOwned + Send + Sync + 'static;
Expand All @@ -28,6 +31,8 @@ pub trait Storable: Debug + Borrow<Self::Key> + Send + Sync + 'static {
cache: PathBuf,
metadata: Self::Metadata,
) -> BoxFuture<'static, Result<Self::Output>>;

fn key(&self) -> Self::Key;
}

#[derive(Clone, Debug, Serialize, Deserialize)]
Expand All @@ -48,13 +53,14 @@ impl<'a, D: Digest> Hasher for HashToDigest<'a, D> {
}

pub fn digest<S: Storable>(storable: &S) -> Result<String> {
let key: &S::Key = storable.borrow();
let key_serialized = bincode::serialize(key)?;
let key = storable.key();
let key_serialized = bincode::serialize(&key)?;

let mut digest = sha2::Sha224::default();
sha2::Digest::update(&mut digest, &[VERSION]);
sha2::Digest::update(&mut digest, &key_serialized);
std::any::TypeId::of::<S::Key>().hash(&mut HashToDigest(&mut digest));
std::any::TypeId::of::<S>().hash(&mut HashToDigest(&mut digest));
let digest = digest.finalize();
Ok(data_encoding::BASE64URL_NOPAD.encode(&digest))
}
Expand Down Expand Up @@ -87,24 +93,29 @@ impl Cache {

let retrieve = async {
let info = entry_meta.read_to_json::<EntryIndex<S>>()?;
crate::fs::require_exist(&entry_dir)?;
storable.adapt(entry_dir.clone(), info.metadata).await
};

if let Ok(out) = retrieve.await {
debug!("Found in cache, skipping generation.");
Ok(out)
} else {
debug!("Not found in cache, will generate.");
let key: &S::Key = storable.borrow();
let info = EntryIndex::<S> {
metadata: storable
.generate(this, entry_dir.clone())
.instrument(info_span!("Generating value to be cached.", ?key))
.await?,
key: key.clone(),
};
entry_meta.write_as_json(&info)?;
storable.adapt(entry_dir, info.metadata).await
match retrieve.await {
Ok(out) => {
debug!("Found in cache, skipping generation.");
return Ok(out);
}
Err(e) => {
debug!("Value cannot be retrieved from cache because: {e}");
crate::fs::reset_dir(&entry_dir)?;
let key = storable.key();
let info = EntryIndex::<S> {
metadata: storable
.generate(this, entry_dir.clone())
.instrument(info_span!("Generating value to be cached.", ?key))
.await?,
key: key.clone(),
};
entry_meta.write_as_json(&info)?;
storable.adapt(entry_dir, info.metadata).await
}
}
}
.boxed()
Expand All @@ -125,8 +136,6 @@ mod tests {

let cache = Cache::new("C:/temp/enso-cache").await?;
cache.get(download_task).await?;


Ok(())
}
}
Loading

0 comments on commit 2fbb1bc

Please sign in to comment.