Merge branch 'master' of https://github.com/brainlife/amaretti

.codespellrc

@@ -0,0 +1,4 @@
+[codespell]
+skip = .git,*.pdf,*.svg,package-lock.json,*.min.js,vendor,.sass-cache,locales,ui-text.yml
+# hsi - server name
+ignore-words-list = keypair,nd,nin,hsi

.github/workflows/codespell.yml

@@ -0,0 +1,19 @@
+---
+name: Codespell
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v1

README.md

@@ -3,7 +3,7 @@
 
 # Amaretti
 
-Amaretti is a light-weight, cross-resource, multi-user task orchestration service written in nodejs. Amaretti is a meta processing managment system; it provides cross-resource orchestration relying on the individual resources batch scheduling systems. It manages data and computation execution across multiple architectures (e.g., clouds and high-performance computing). Its purpose is to reduce the users' management burden, facilitate data-intensive research and development and accelerate discovery.
+Amaretti is a light-weight, cross-resource, multi-user task orchestration service written in nodejs. Amaretti is a meta processing management system; it provides cross-resource orchestration relying on the individual resources batch scheduling systems. It manages data and computation execution across multiple architectures (e.g., clouds and high-performance computing). Its purpose is to reduce the users' management burden, facilitate data-intensive research and development and accelerate discovery.
 
 For more information, please read [Amaretti Doc](https://brain-life.github.io/amaretti/)
 

api/common.js

@@ -113,7 +113,7 @@ exports.encrypt_resource = function(resource) {
 }
 
 //decrypt all config parameter that starts with enc_
-//TODO crypto could throw execption - but none of the client are handling it.. 
+//TODO crypto could throw exception - but none of the client are handling it.. 
 exports.decrypt_resource = function(resource) {
     if(resource.decrypted) {
         console.log("resource already decrypted");
@@ -469,7 +469,7 @@ exports.check_access = function(user, resource) {
 exports.canUseResource = function(user, resource) {
     const sub = user.sub.toString();
 
-    //ownwer/admin can access the resource
+    //owner/admin can access the resource
     if(resource.user_id == sub) return true;
     if(resource.admins.includes[sub]) return true;
 
@@ -625,7 +625,7 @@ exports.rerun_task = function(task, remove_date, cb) {
             //TODO - rerunning task with active deps might not be always bad - if the tasks are in different resource.
             //let's not veto this completely.. I need to think more
             console.log("rerunning task with active deps - it might make the deps fail");
-            //return cb("Can't rerun this task as it has dependent tasks that are currrently running.");
+            //return cb("Can't rerun this task as it has dependent tasks that are currently running.");
         }
 
         //all good! let's proceed with rerunning

api/controllers/admin.js

@@ -32,7 +32,7 @@ router.get('/services/running', common.jwt(), function(req, res, next) {
                 {status: "running"},
 
                 //also include requests that are currently getting started - so that we don't undercount tasks
-                //while deciding how many are commited to run on various resources
+                //while deciding how many are committed to run on various resources
                 {status: "requested", start_date: {$exists: true}}, 
             ]
         }},

api/controllers/index.js

@@ -56,7 +56,7 @@ router.use('/taskevent', require('./taskevent'));
 //use (get) /resource/type instead
 router.get('/config', common.jwt({credentialsRequired: false}), function(req, res) {
     var conf = {
-        resources: config.resources, //resoruce types
+        resources: config.resources, //resource types
     };
     res.json(conf);
 });

api/controllers/task.js

@@ -950,7 +950,7 @@ router.put('/stop/:task_id', common.jwt(), function(req, res, next) {
         }
 
         //TODO - _handled is deprecated, but I should still make sure that the task isn't currently handled? but how?
-        //if(task._handled) return next("The task is currently handled by sca-task serivce. Please wait..");
+        //if(task._handled) return next("The task is currently handled by sca-task service. Please wait..");
 
         switch(task.status) {
         case "running":

api/events.js

@@ -50,7 +50,7 @@ exports.init = function(cb) {
         //
         /////////////////////////////////////////////////////////////
 
-        //I am not sure if ready event fires everytime it reconnects.. (find out!) 
+        //I am not sure if ready event fires every time it reconnects.. (find out!) 
         //so let's clear cb() once I call it
         if(cb) {
             cb();

api/models.js

@@ -283,7 +283,7 @@ var taskSchema = mongoose.Schema({
     fail_date: Date, //time when this task was last failed
     update_date: Date, //time when this task was last updated (only used by put api?)
     request_date: Date, //time when this task was requested (!=create_date if re-requested)
-    remove_date: Date, //date when the task dir should be removed (if not requested or running) - if not set, will be remved after 25 days
+    remove_date: Date, //date when the task dir should be removed (if not requested or running) - if not set, will be removed after 25 days
 
     handle_date: {type: Date, default: Date.now }, //last time this task was handled by task handler
 

api/resource.js

@@ -263,7 +263,7 @@ function check_ssh(resource, cb) {
 
             if(err) return cb_once(err);
             let to = setTimeout(()=>{
-                cb_once(null, "failed", "send test script timeout(10sec) - filesytem is offline?");
+                cb_once(null, "failed", "send test script timeout(10sec) - filesystem is offline?");
                 to = null;
             }, 10*1000); 
 
@@ -372,7 +372,7 @@ function check_iohost(resource, cb) {
 
             if(err) return cb_once(err);
             let to = setTimeout(()=>{
-                cb_once(null, "failed", "readdir timeout - filesytem is offline?");
+                cb_once(null, "failed", "readdir timeout - filesystem is offline?");
                 to = null;
             }, 3*1000);
             sftp.opendir(workdir, function(err, stat) {
@@ -417,7 +417,7 @@ exports.stat = async function(resource, cb) {
     try {
         //get execution history counts for each service 
         let data = await db.Taskevent.aggregate()
-            .match({resource_id: resource._id, status: {$in: ["running", "finished", "failed"]}}) //reuqested doesn't get resource_id
+            .match({resource_id: resource._id, status: {$in: ["running", "finished", "failed"]}}) //requested doesn't get resource_id
             .group({_id: {service: '$service', status: '$status'}, count: {$sum: 1}}).exec()
         let total = {};
         let services = {};

api/resource_test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#This script will be copied to each resource everytime resource checker test resource status and used to test the resource
+#This script will be copied to each resource every time resource checker test resource status and used to test the resource
 
 #check for common binaries
 which git >/dev/null

api/service.js

@@ -20,7 +20,7 @@ exports.loaddetail = function(service_name, branch, cb) {
     var cache = _details_cache[service_name];
     var now = new Date();
     if(cache) {
-        //check for expiriation
+        //check for expiration
         var age = new Date() - cache.date;
         if(age > 1000*60*10) { //10 minutes
             //expired
@@ -155,7 +155,7 @@ exports.get_sha = function(service_name, branch, cb) {
     var cache = _sha_cache[name];
     var now = new Date();
     if(cache) {
-        //check for expiriation
+        //check for expiration
         var age = new Date() - cache.date;
         if(age > 1000*10) { //cache for 10 seconds 
             //expired

api/transfer.js

@@ -73,7 +73,7 @@ exports.rsync_resource = function(source_resource, dest_resource, source_path, d
                     console.debug("source_path", source_path, source_id);
                     console.debug("dest_path", dest_path, dest_id);
                     if(dest_id == source_id) {  
-                        console.log("it *looks* like source filesytem is the same as dest filesytem.. skipping sync");
+                        console.log("it *looks* like source filesystem is the same as dest filesystem.. skipping sync");
                         return cb(); //we are all done!
                     }
                     next();
@@ -84,7 +84,7 @@ exports.rsync_resource = function(source_resource, dest_resource, source_path, d
         //cleanup broken symlinks on source resource
         //also check for infinite loop
         next=>{
-            //we are using rsync -L to derefernce symlink, which would fail if link is broken. so this is an ugly 
+            //we are using rsync -L to dereference symlink, which would fail if link is broken. so this is an ugly 
             //workaround for rsync not being forgivng..
             //console.log("finding and removing broken symlink on source resource before rsync", source_path);
             common.get_ssh_connection(source_resource, {}, (err, conn)=>{
@@ -106,7 +106,7 @@ exports.rsync_resource = function(source_resource, dest_resource, source_path, d
                             if(err) return next(err);
                             stream.on('close', (code, signal)=>{
                                 if(code === undefined) return next("connection closed while detecting infinite sym loop");
-                                else if(code) return next("filesytem loop detected");
+                                else if(code) return next("filesystem loop detected");
                                 next();
                             })
                             .on('data', data=>{
@@ -140,7 +140,7 @@ exports.rsync_resource = function(source_resource, dest_resource, source_path, d
             const source = source_resource.config.username+"@"+source_hostname+":"+source_path+"/";
 
             //-v writes output to stderr.. even though it's not error..
-            //-L is to follow symlinks (addtionally) --safe-links is desirable, but it will not transfer inter task/instance symlinks
+            //-L is to follow symlinks (additionally) --safe-links is desirable, but it will not transfer inter task/instance symlinks
             //-e opts is for ssh
             //-h is to make it human readable
 

bin/metrics.js

@@ -18,7 +18,7 @@ const influx = require('@influxdata/influxdb-client');
 let graphite_prefix = process.argv[2];
 if(!graphite_prefix) graphite_prefix = "dev";
 
-//TODO - this has to match up between amaretti/bin/metrics and warehouse/api/controller querying for graphite daa
+//TODO - this has to match up between amaretti/bin/metrics and warehouse/api/controller querying for graphite data
 function sensu_name(name) {
     name = name.toLowerCase();
     name = name.replace(/[_.@$#\/]/g, "-");

bin/remove_workdirs.js

@@ -54,7 +54,7 @@ function remove(task) {
         let old = new Date();
         old.setMonth(-6);
         if(task.create_date < old) {
-            console.debug("task was created very long time ago.. but still trying to remove workdir.. probably the resource used disappeard and got stuck.. clearing");
+            console.debug("task was created very long time ago.. but still trying to remove workdir.. probably the resource used disappeared and got stuck.. clearing");
             task.resource_ids = [];
             return resolve();
         }

bin/serviceinfo.js

@@ -130,7 +130,7 @@ db.init(function(err) {
 
                     let runtimes = [];
                     async.eachSeries(finish_events, (finish_event, next_finish_event)=>{
-                        //find when it started running for tha task
+                        //find when it started running for that task
                         db.Taskevent.findOne({
                             service: k, 
                             status: {$in: ["running", "running_sync"]}, 

bin/task.js

@@ -187,7 +187,7 @@ function check(cb) {
 }
 
 function handle_housekeeping(task, cb) {
-    //console.debug("houskeeping!");
+    //console.debug("housekeeping!");
     async.series([
 
         //for validator (with follow_task_id) if the parent task is removed, we should mark the task as removed also
@@ -349,7 +349,7 @@ async function handle_requested(task, next) {
     //requested jobs are handled asynchronously.. (start_date will be set while being handled)
     //if some api reset next_date, it could get reprocessed while it's starting up
     //so we need to bail if this is the cause
-    //WARNING - don't run anything asynchrnous after checking for task.start_date before I save the task with new start_date 
+    //WARNING - don't run anything asynchronous after checking for task.start_date before I save the task with new start_date 
     if(task.start_date) {
         let starting_for = now - task.start_date;
         //console.log("start_date is set", starting_for);
@@ -495,7 +495,7 @@ async function handle_requested(task, next) {
 
         //Don't wait for start_task to finish.. could take a while to start.. (especially rsyncing could take a while).. 
         //start_task is designed to be able to run concurrently..
-        console.log("started task.. skiping save");
+        console.log("started task.. skipping save");
         next(null, true); //skip saving to prevent parallel save with start_task
     });
 }
@@ -650,7 +650,7 @@ function handle_running(task, next) {
                             next();
                             break;
                         case 1: //finished
-                            //I am not sure if I have enough use cases to warrent the automatical retrieval of product.json to task..
+                            //I am not sure if I have enough use cases to warrant the automatic retrieval of product.json to task..
                             console.debug("finished!");
                             load_product(taskdir, resource, async (err, product)=>{
                                 if(err) {
@@ -661,7 +661,7 @@ function handle_running(task, next) {
                                     next();
                                 } else {
                                     task.finish_date = new Date();
-                                    if(!task.start_date) task.start_date = task.create_date; //shoudn't happen, but it does sometimes.
+                                    if(!task.start_date) task.start_date = task.create_date; //shouldn't happen, but it does sometimes.
                                     task.runtime = task.finish_date.getTime() - task.start_date.getTime();
                                     task.status = "finished";
                                     task.status_msg = "Successfully completed in "+(task.runtime/(1000*60)).toFixed(2)+" mins on "+resource.name;
@@ -678,7 +678,7 @@ function handle_running(task, next) {
                             task.fail_date = new Date();
                             poke_next(task, next);
                             break;
-                        case 3: //status temporarly unknown
+                        case 3: //status temporarily unknown
                             //TODO - I should mark the job as failurer if it's been a long time since last good status output
                             //of 3 stries and out?
                             console.error("couldn't determine the job state. could be an issue with status script on resource:%s", resource.name, task.instance_id+"/"+task._id);
@@ -1357,7 +1357,7 @@ async function storeProduct(task, dirty_product) {
                 });
             }
 
-            product = deepmerge(product, follow_product.product); //TODO shouldn't product have precidence over follow_product?
+            product = deepmerge(product, follow_product.product); //TODO shouldn't product have precedence over follow_product?
         }
     }
     if(!product) return;

docs/_config.yml

@@ -145,7 +145,7 @@ markdown_ext: "markdown,mkdown,mkdn,mkd,md"
 
 # Conversion
 markdown: kramdown
-highlighter: rouge
+highlighter: rogue
 lsi: false
 excerpt_separator: "\n\n"
 incremental: false

docs/_pages/contact.md

@@ -6,8 +6,8 @@ sidebar:
   nav: "docs"
 ---
 
-### [Soichi Hayashi](mailto:hayashis@iu.edu) (Software Engineer)
+### Brainlife Support Team: support@brainlife.io
 
-### [Franco Pestilli](mailto:frakkopesto@gmail.com) (Brain-Life PI)
+### Franco Pestilli, Brainlife PI: pestilli@utexas.edu