Skip to content

Commit

Permalink
Merge pull request #76 from JarvusInnovations/develop
Browse files Browse the repository at this point in the history 
Release: hologit v0.28.0
  • Loading branch information
@themightychris
themightychris authored Dec 24, 2019
2 parents ace2648 + 7d8e4c6 commit 51da77c
Show file tree
Hide file tree
Showing 6 changed files with 173 additions and 104 deletions.
20 changes: 2 additions & 18 deletions commands/lens/exec.js
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,10 @@
exports.command = 'exec <spec-hash>';
exports.desc = 'Execute lens spec and output result hash';
exports.builder = {
refresh: {
describe: 'True to execute lens even if existing result is available',
type: 'boolean',
default: false
},
save: {
describe: 'False to disable updating spec ref with result',
type: 'boolean',
default: true
}
};

exports.handler = async function exportTree ({
specHash,
refresh=false,
save=true
}) {
exports.handler = async function exportTree ({ specHash }) {
const Lens = require('../../lib/Lens.js');

const lensedTreeHash = await Lens.executeSpec(specHash, { refresh, save });
const lensedTreeHash = await Lens.executeSpec(specHash, { refresh: true });

if (lensedTreeHash) {
console.log(lensedTreeHash);
Expand Down
44 changes: 41 additions & 3 deletions commands/project.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ exports.builder = {
describe: 'Set to continously output updated output',
type: 'boolean',
default: false
},
'cache-to': {
describe: 'Set a remote to push caches to',
type: 'string'
},
'cache-from': {
describe: 'Set a remote to pull caches from',
type: 'string'
}
};

Expand All @@ -44,7 +52,9 @@ exports.handler = async function project ({
fetch = null,
watch = false,
commitTo = null,
commitMessage = null
commitMessage = null,
cacheFrom = null,
cacheTo = null
}) {
const logger = require('../lib/logger.js');
const { Repo, Projection } = require('../lib');
Expand Down Expand Up @@ -77,6 +87,30 @@ exports.handler = async function project ({
}


// parse cache from/to from env
if (cacheFrom === false) {
cacheFrom = null;
} else if (!cacheFrom) {
const cacheFromEnv = process.env.HOLO_CACHE_FROM;
if (cacheFromEnv) {
cacheFrom = cacheFromEnv;
} else {
cacheFrom = 'origin';
}
} else if (typeof cacheFrom != 'string') {
throw new Error(`cache-from must be a single string value`);
}

if (!cacheTo) {
const cacheToEnv = process.env.HOLO_CACHE_TO;
if (cacheToEnv) {
cacheTo = cacheToEnv;
}
} else if (typeof cacheTo != 'string') {
throw new Error(`cache-to must be a single string value`);
}


// load holorepo
const repo = await Repo.getFromEnvironment({ ref, working });
const parentCommit = await repo.resolveRef();
Expand All @@ -103,7 +137,9 @@ exports.handler = async function project ({
commitTo,
commitMessage,
parentCommit,
fetch
fetch,
cacheFrom,
cacheTo
});
console.log(outputHash);

Expand All @@ -120,7 +156,9 @@ exports.handler = async function project ({
lens,
commitTo,
commitMessage,
parentCommit: newCommitHash
parentCommit: newCommitHash,
cacheFrom,
cacheTo
});
console.log(outputHash);
}
Expand Down
6 changes: 4 additions & 2 deletions lib/Branch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,9 @@ class Branch extends Configurable {

async composite ({
outputTree = this.getRepo().createTree(),
fetch = false
fetch = false,
cacheFrom = null,
cacheTo = null
}) {
const repo = this.getRepo();
const mappings = await this.getMappings();
Expand Down Expand Up @@ -232,7 +234,7 @@ class Branch extends Configurable {
}

// load tree
const sourceTreeHash = await source.getOutputTree({ fetch });
const sourceTreeHash = await source.getOutputTree({ fetch, cacheFrom, cacheTo });
const sourceTree = await repo.createTreeFromRef(`${sourceTreeHash}:${root == '.' ? '' : root}`);

// merge source into target
Expand Down
175 changes: 112 additions & 63 deletions lib/Lens.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,13 +189,7 @@ class Lens extends Configurable {
return Lens.executeSpec(specHash, {...options, repo: this.workspace.getRepo()});
}

static async executeSpec (specHash, { refresh=false, save=true, repo=null }) {

// return studio.holoLensExec(specHash);

// TODO: run in studio if not in one?
// TODO: delegate some of this to Studio

static async executeSpec (specHash, { refresh=false, save=true, repo=null, cacheFrom=null, cacheTo=null }) {

// load holorepo
if (!repo) {
Expand All @@ -205,92 +199,111 @@ class Lens extends Configurable {
const git = await repo.getGit();


// ensure the rest runs inside a studio environment
if (!await Studio.isEnvironmentStudio()) {
const studio = await Studio.get(repo.gitDir);
return studio.holoLensExec(specHash);
}


// check for existing build
const specRef = SpecObject.buildRef('lens', specHash);
if (!refresh) {
const existingBuildHash = await repo.resolveRef(specRef);
let existingBuildHash = await git.getTreeHash(specRef, { verify: false });

if (existingBuildHash) {
logger.info(`found existing output tree matching holospec(${specHash})`);

if (cacheTo) {
await _cacheResultTo(repo, specRef, cacheTo);
}

return existingBuildHash;
}
}

if (cacheFrom) {
existingBuildHash = await _cacheResultFrom(repo, specRef, cacheFrom);

// load spec
const specToml = await git.catFile({ p: true }, specHash);
const {
holospec: {
lens: spec
if (existingBuildHash) {
return existingBuildHash;
}
}
} = TOML.parse(specToml);
}


// assign scratch directory
const scratchPath = `${process.env.HOLO_SCRATCH||'/hab/cache/hololens'}/${spec.package.split('/').slice(0, 2).join('/')}`;
await mkdirp(scratchPath);
// ensure the rest runs inside a studio environment
let lensedTreeHash;

if (!await Studio.isEnvironmentStudio()) {
const studio = await Studio.get(repo.gitDir);
lensedTreeHash = await studio.holoLensExec(specHash);
} else {
// load spec
const specToml = await git.catFile({ p: true }, specHash);
const {
holospec: {
lens: spec
}
} = TOML.parse(specToml);

// install lens package
const hab = await Studio.getHab();
logger.info(`installing lens package: ${spec.package}`);
await hab.pkg('install', spec.package);

// assign scratch directory
const scratchPath = `${process.env.HOLO_SCRATCH||'/hab/cache/hololens'}/${spec.package.split('/').slice(0, 2).join('/')}`;
await mkdirp(scratchPath);

// prepare command and environment
const command = handlebars.compile(spec.command)(spec);
const env = Object.assign(
squish(
{
hololens: { ...spec, spec: specHash }
},

// install lens package
const hab = await Studio.getHab();
logger.info(`installing lens package: ${spec.package}`);
await hab.pkg('install', spec.package);


// prepare command and environment
const command = handlebars.compile(spec.command)(spec);
const env = Object.assign(
squish(
{
hololens: { ...spec, spec: specHash }
},
{
seperator: '_',
modifyKey: key => key.toUpperCase().replace(/-/g, '_')
}
),
{
seperator: '_',
modifyKey: key => key.toUpperCase().replace(/-/g, '_')
GIT_DIR: repo.gitDir,
GIT_WORK_TREE: scratchPath,
GIT_INDEX_FILE: `${scratchPath}.index`,
DEBUG: process.env.DEBUG || ''
}
),
{
GIT_DIR: repo.gitDir,
GIT_WORK_TREE: scratchPath,
GIT_INDEX_FILE: `${scratchPath}.index`,
DEBUG: process.env.DEBUG || ''
}
);
logger.debug('lens environment:', env);
);
logger.debug('lens environment:', env);


// spawn process and log STDERR
logger.info(`executing lens command: ${command}`);
const lensProcess = await hab.pkg('exec', spec.package, ...shellParse(command), {
$env: env,
$cwd: scratchPath,
$spawn: true
});
// spawn process and log STDERR
logger.info(`executing lens command: ${command}`);
const lensProcess = await hab.pkg('exec', spec.package, ...shellParse(command), {
$env: env,
$cwd: scratchPath,
$spawn: true
});

lensProcess.stderr.on('data', data => {
data.toString().trim().split(/\n/).forEach(
line => logger.info(`lens: ${line}`)
)
});
lensProcess.stderr.on('data', data => {
data.toString().trim().split(/\n/).forEach(
line => logger.info(`lens: ${line}`)
)
});


// process output
const lensedTreeHash = await lensProcess.captureOutputTrimmed();
// process output
lensedTreeHash = await lensProcess.captureOutputTrimmed();

if (!git.isHash(lensedTreeHash)) {
throw new Error(`lens "${command}" did not return hash: ${lensedTreeHash}`);
if (!git.isHash(lensedTreeHash)) {
throw new Error(`lens "${command}" did not return hash: ${lensedTreeHash}`);
}
}


// save ref to accelerate next projection
if (save) {
await git.updateRef(specRef, lensedTreeHash);

if (cacheTo) {
await _cacheResultTo(repo, specRef, cacheTo);
}
}


Expand All @@ -301,3 +314,39 @@ class Lens extends Configurable {
}

module.exports = Lens;


// private methods
function _getTrackingRef(specRef, remote) {
return `refs/holo/lens-remotes/${remote}/${specRef.substr(15)}`;
}

async function _cacheResultFrom(repo, specRef, cacheFrom) {
const git = await repo.getGit();

try {
await git.fetch(cacheFrom, `${specRef}:${specRef}`);

const existingBuildHash = await git.getTreeHash(specRef, { verify: false });

if (existingBuildHash) {
logger.info(`fetched cached result from ${cacheFrom}: ${specRef}`);
await git.updateRef(_getTrackingRef(specRef, cacheFrom), existingBuildHash);
return existingBuildHash;
}
} catch (err) {
return null;
}
}


async function _cacheResultTo(repo, specRef, cacheTo) {
const git = await repo.getGit();
const trackingSpecRef = _getTrackingRef(specRef, cacheTo);

if (!await repo.resolveRef(trackingSpecRef)) {
logger.info(`pushing cached result to ${cacheTo}: ${specRef}`);
await git.push(cacheTo, specRef);
await git.updateRef(trackingSpecRef, specRef);
}
}
Loading

0 comments on commit 51da77c

Please sign in to comment.