From f9688b0c1578c0bb4a3596ebc2d44ac597fcbbf2 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 8 May 2017 20:58:05 +0000
Subject: [PATCH 01/43] Add maintenance infastructure

---
 kernel-lib/services/php-fpm.js | 25 ++++++++++++++++
 kernel-lib/sites.js            | 20 +++++++++++++
 php-bootstrap/maintenance.php  | 55 ++++++++++++++++++++++++++++++++++
 3 files changed, 100 insertions(+)
 create mode 100644 php-bootstrap/maintenance.php

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 008e021..7edcb81 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -46,6 +46,9 @@ exports.PhpFpmService = function(name, controller, options) {
 
     // listen for site updated
     controller.sites.on('siteUpdated', _.bind(me.onSiteUpdated, me));
+
+    // listen for maintenance requests
+    controller.sites.on('maintenanceRequested', _.bind(me.onMaintenanceRequested, me));
 };
 
 util.inherits(exports.PhpFpmService, require('./abstract.js').AbstractService);
@@ -221,3 +224,25 @@ exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) {
         if (phpErrors) console.error(phpErrors);
     });
 };
+
+exports.PhpFpmService.prototype.onMaintenanceRequested = function(siteData) {
+    var me = this,
+        phpClient;
+
+    // Connect to FPM worker pool
+    phpClient = new phpfpm({
+        sockFile: me.options.socketPath,
+        documentRoot: me.options.bootstrapDir + '/'
+    });
+
+    // Run maintenance request
+    phpClient.run({
+        uri: 'maintenance.php',
+        json: siteData
+    }, function(err, output, phpErrors) {
+        if (err == 99) console.error('PHPFPM server error');
+        console.log(output);
+        // @todo update given jobs
+        if (phpErrors) console.error(phpErrors);
+    });
+}
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 956d58f..6910ea2 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -203,6 +203,26 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     });
 
                     return true;
+
+                } else if (request.path[2] == 'maintenance') {
+
+                    // append site handle to all commands
+                    for (i=0; i<requestData.length; i++) {
+                        requestData[i].handle = request.path[1];
+                    }
+
+                    console.log('Received maintenance request for ' + request.path[1]);
+                    console.log(requestData);
+
+                    me.emit('maintenanceRequested', requestData);
+
+                    response.writeHead(200, {'Content-Type':'application/json'});
+                    response.end(JSON.stringify({
+                        success: true,
+                        message: 'maintenance request finished',
+                    }));
+                    return true;
+
                 } else {
                     console.error('Unhandled site sub-resource: ' + request.path[2]);
                     response.writeHead(404, {'Content-Type':'application/json'});
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
new file mode 100644
index 0000000..2f28f64
--- /dev/null
+++ b/php-bootstrap/maintenance.php
@@ -0,0 +1,55 @@
+<?php
+
+// bootstrap emergence
+require('bootstrap.inc.php');
+
+// only process POST requests
+if ($_SERVER['REQUEST_METHOD'] != 'POST') {
+    exit();
+}
+
+$commands = json_decode(file_get_contents('php://input'), true);
+
+foreach ($commands as $command) {
+    switch ($command['action'])
+    {
+        case 'cache':
+            handleCacheRequest($command);
+            break;
+        case 'vfs':
+            handleVFSRequest($command);
+            break;
+    }
+}
+
+// Create, update or delete cached elements
+function handleCacheRequest($command)
+{
+    // Raw delete
+    if (!empty($command['delete'])) {
+        Cache::rawDelete($command['handle'] . ':' . $command['delete']);
+    }
+
+    // Pattern delete
+    // Input should not include handle or slashes
+    // Example: pattern 'abc' will delete pattern '/^HANDLE:abc/'
+    if (!empty($command['pattern'])) {
+        foreach (CacheIterator::createFromPattern('/^' . $command['handle'] . ':' . $command['pattern'] . '/') as $cacheEntry) {
+            Cache::rawDelete($cacheEntry['key']);
+        }
+    }
+
+    // Create entry
+    if (!empty($command['create'])) {
+        Cache::rawStore(
+            $command['handle'] . ':' . $command['create'],
+            empty($command['value']) ? null : $command['value'],
+            empty($command['ttl']) ? 0 : $command['ttl']
+        );
+    }
+}
+
+function handleVFSRequest($command)
+{
+    
+}
\ No newline at end of file

From 9459a55655f9221d695fddfd128cc0d4520c4e6f Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 10 May 2017 17:26:26 +0000
Subject: [PATCH 02/43] Add vfs update command

---
 kernel-lib/services/php-fpm.js |  11 ++-
 kernel-lib/sites.js            |   7 +-
 php-bootstrap/maintenance.php  | 122 ++++++++++++++++++++++++++++++---
 3 files changed, 121 insertions(+), 19 deletions(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 7edcb81..642c263 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -213,6 +213,7 @@ exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) {
     });
 
     // Clear cached site.json
+    // @todo update to use maintenance.php instead of cache.php
     phpClient.run({
         uri: 'cache.php',
         json: [
@@ -225,12 +226,14 @@ exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) {
     });
 };
 
-exports.PhpFpmService.prototype.onMaintenanceRequested = function(siteData) {
+exports.PhpFpmService.prototype.onMaintenanceRequested = function(siteData, handle) {
     var me = this,
+        siteRoot = me.controller.sites.options.sitesDir + '/' + handle,
         phpClient;
 
     // Connect to FPM worker pool
     phpClient = new phpfpm({
+        handle: handle,
         sockFile: me.options.socketPath,
         documentRoot: me.options.bootstrapDir + '/'
     });
@@ -238,7 +241,11 @@ exports.PhpFpmService.prototype.onMaintenanceRequested = function(siteData) {
     // Run maintenance request
     phpClient.run({
         uri: 'maintenance.php',
-        json: siteData
+        json: {
+            'commands': siteData,
+            'handle': handle,
+            'siteRoot': siteRoot
+        }
     }, function(err, output, phpErrors) {
         if (err == 99) console.error('PHPFPM server error');
         console.log(output);
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 6910ea2..bdf5b2f 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -206,15 +206,10 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
                 } else if (request.path[2] == 'maintenance') {
 
-                    // append site handle to all commands
-                    for (i=0; i<requestData.length; i++) {
-                        requestData[i].handle = request.path[1];
-                    }
-
                     console.log('Received maintenance request for ' + request.path[1]);
                     console.log(requestData);
 
-                    me.emit('maintenanceRequested', requestData);
+                    me.emit('maintenanceRequested', requestData, request.path[1]);
 
                     response.writeHead(200, {'Content-Type':'application/json'});
                     response.end(JSON.stringify({
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
index 2f28f64..aa72d8f 100644
--- a/php-bootstrap/maintenance.php
+++ b/php-bootstrap/maintenance.php
@@ -8,33 +8,33 @@
     exit();
 }
 
-$commands = json_decode(file_get_contents('php://input'), true);
+$data = json_decode(file_get_contents('php://input'), true);
 
-foreach ($commands as $command) {
+foreach ($data['commands'] as $command) {
     switch ($command['action'])
     {
         case 'cache':
-            handleCacheRequest($command);
+            handleCacheRequest($command, $data['handle'], $data['siteRoot']);
             break;
         case 'vfs':
-            handleVFSRequest($command);
+            handleVFSRequest($command, $data['handle'], $data['siteRoot']);
             break;
     }
 }
 
 // Create, update or delete cached elements
-function handleCacheRequest($command)
+function handleCacheRequest($command, $handle, $siteRoot)
 {
     // Raw delete
     if (!empty($command['delete'])) {
-        Cache::rawDelete($command['handle'] . ':' . $command['delete']);
+        Cache::rawDelete($handle . ':' . $command['delete']);
     }
 
     // Pattern delete
     // Input should not include handle or slashes
     // Example: pattern 'abc' will delete pattern '/^HANDLE:abc/'
     if (!empty($command['pattern'])) {
-        foreach (CacheIterator::createFromPattern('/^' . $command['handle'] . ':' . $command['pattern'] . '/') as $cacheEntry) {
+        foreach (CacheIterator::createFromPattern('/^' . $handle . ':' . $command['pattern'] . '/') as $cacheEntry) {
             Cache::rawDelete($cacheEntry['key']);
         }
     }
@@ -42,14 +42,114 @@ function handleCacheRequest($command)
     // Create entry
     if (!empty($command['create'])) {
         Cache::rawStore(
-            $command['handle'] . ':' . $command['create'],
+            $handle . ':' . $command['create'],
             empty($command['value']) ? null : $command['value'],
             empty($command['ttl']) ? 0 : $command['ttl']
         );
     }
 }
 
-function handleVFSRequest($command)
+function handleVFSRequest($command, $handle, $siteRoot)
 {
-    
-}
\ No newline at end of file
+    // Initialize site for Emergence FS commands
+    Site::initialize($siteRoot);
+
+    // Update the file system
+    updateFileSystem($handle, $command['cursor'], $siteRoot);
+}
+
+function updateFileSystem($handle, $cursor = 0)
+{
+    // Turn on autopull to enable remote file downloads
+    \Site::$autoPull = true;
+
+    // Get update summary
+    $summary = getFileSystemSummary($cursor);
+
+    // Precache files
+    foreach ($summary['new'] as $path) {
+        $Node = \Site::resolvePath($path);
+    }
+
+    // Delete files
+    foreach ($summary['deleted'] as $path) {
+        if ($Node = \Site::resolvePath($path)) {
+            $Node->delete();
+        }
+    }
+
+    // Update files
+    foreach ($summary['updated'] as $path) {
+        $Node = \Site::resolvePath($path);
+        $NewNode = \Emergence::resolveFileFromParent($Node->Collection, $Node->Handle, true);
+    }
+
+    // Get updated local cursor
+    $summary['localCursor'] = getLocalCursor();
+
+    return $summary;
+}
+
+// Retrieve the file summary for given site
+function getFileSystemSummary($cursor = 0)
+{
+    // Local files / keys
+    $localFiles = Emergence_FS::getTreeFiles(null, false);
+    $localKeys = array_keys($localFiles);
+
+    // Get parent files / keys
+    $parentVFSUrl = Emergence::buildUrl();
+    $curl = curl_init();
+    curl_setopt_array($curl, array(
+        CURLOPT_RETURNTRANSFER => 1,
+        CURLOPT_URL => $parentVFSUrl
+    ));
+    $parentFileResponse = json_decode(curl_exec($curl), true);
+    $parentFiles = $parentFileResponse['files'];
+    $parentKeys = array_keys($parentFiles);
+
+    $newFiles = [];
+    $updatedFiles = [];
+    $deletedFiles = [];
+    $parentCursor = $cursor;
+
+    // Compare remote files against their local copies
+    foreach ($parentFiles as $path => $data) {
+
+        // Update parent cursor
+        if ($data['ID'] > $parentCursor) {
+            $parentCursor = $data['ID'];
+        }
+
+        // @todo instead of comparing against local keys, do real time queries
+        // for local nodes. This will minimize the number of total queries run
+        // and won't require the connection to the local /emergence endpoint
+
+        // Find new files
+        if (!in_array($path, $localKeys)) {
+            array_push($newFiles, $path);
+
+        // Find deleted files
+        } elseif ($data['SHA1'] == null && $localFiles[$path]['Site'] == 'Remote') {
+            array_push($deletedFiles, $path);
+
+        // Find updated files by mismatched SHA1s
+        } elseif ($data['SHA1'] !== $localFiles[$path]['SHA1'] && $localFiles[$path]['Site'] == 'Remote') {
+            array_push($updatedFiles, $path);
+        }
+    }
+
+    return [
+        'new' => $newFiles,
+        'updated' => $updatedFiles,
+        'deleted' => $deletedFiles,
+        'parentCursor' => $parentCursor,
+        'localCursor' => getLocalCursor()
+    ];
+}
+
+// Retrieve the local cursor for the given site
+function getLocalCursor()
+{
+    return \DB::oneValue('SELECT MAX(ID) FROM _e_files');
+}

From 22d97484d49552fe989a2edd7f13b3d9ad0dde6d Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 13:57:17 +0000
Subject: [PATCH 03/43] Add maintenance request to jobs stored in memory

---
 kernel-lib/services/php-fpm.js | 14 +++++---
 kernel-lib/sites.js            | 60 ++++++++++++++++++++++++++++++++--
 php-bootstrap/maintenance.php  | 30 +++++++++++++----
 3 files changed, 92 insertions(+), 12 deletions(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 642c263..f0fc26d 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -226,7 +226,7 @@ exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) {
     });
 };
 
-exports.PhpFpmService.prototype.onMaintenanceRequested = function(siteData, handle) {
+exports.PhpFpmService.prototype.onMaintenanceRequested = function(job, handle) {
     var me = this,
         siteRoot = me.controller.sites.options.sitesDir + '/' + handle,
         phpClient;
@@ -242,14 +242,20 @@ exports.PhpFpmService.prototype.onMaintenanceRequested = function(siteData, hand
     phpClient.run({
         uri: 'maintenance.php',
         json: {
-            'commands': siteData,
+            'job': job,
             'handle': handle,
             'siteRoot': siteRoot
         }
     }, function(err, output, phpErrors) {
         if (err == 99) console.error('PHPFPM server error');
-        console.log(output);
-        // @todo update given jobs
         if (phpErrors) console.error(phpErrors);
+
+        // Parse job response
+        var response = JSON.parse(output);
+
+        // Update job with response
+        job.commands = response.commands;
+        job.status = 'completed';
+        job.completed = new Date().getTime();
     });
 }
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index bdf5b2f..e68ed7d 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -65,6 +65,25 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                 return;
             }
 
+            if (request.path[2]) {
+                if (request.path[2] == 'maintenance') {
+                    console.log('Received maintenance GET request for ' + request.path[1]);
+                    response.writeHead(200, {'Content-Type':'application/json'});
+                    response.end(JSON.stringify({
+                        success: true,
+                        message: 'maintenance request finished',
+                        jobs: me.sites[request.path[1]].jobs
+                    }));
+                    return true;
+
+                } else {
+                    console.error('Unhandled site sub-resource: ' + request.path[2]);
+                    response.writeHead(404, {'Content-Type':'application/json'});
+                    response.end(JSON.stringify({success: false, message: 'Site resource not found'}));
+                    return;
+                }
+            }
+
             response.writeHead(200, {'Content-Type':'application/json'});
             response.end(JSON.stringify({data: me.sites[request.path[1]]}));
             return true;
@@ -208,13 +227,50 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
                     console.log('Received maintenance request for ' + request.path[1]);
                     console.log(requestData);
+                    console.log('Existing jobs');
+                    console.log( me.sites[request.path[1]]['jobs']);
 
-                    me.emit('maintenanceRequested', requestData, request.path[1]);
+                    // Init jobs array
+                    if (typeof(me.sites[request.path[1]]['jobs']) == 'undefined') {
+                        me.sites[request.path[1]]['jobs'] = {};
+                    }
+
+                    // Clean up completed jobs
+                    var jobKeys = Object.keys(me.sites[request.path[1]]['jobs']),
+                        cutoffTime = new Date().getTime() - (60 * 60 * 1000); // 1 hour ago
+                    for (i=0; i<jobKeys.length; i++) {
+                        if (me.sites[request.path[1]]['jobs'][jobKeys[i]].completed && me.sites[request.path[1]]['jobs'][jobKeys[i]].completed < cutoffTime) {
+                            console.log('Remove job');
+                            console.log(me.sites[request.path[1]]['jobs'][jobKeys[i]]);
+                            delete me.sites[request.path[1]]['jobs'][jobKeys[i]];
+                        }
+                    }
+
+                    // Create uid
+                    var uid = Math.floor((Math.random() * 100000));
+                    while (Object.keys(me.sites[request.path[1]]['jobs']).indexOf(uid) !== -1) {
+                        uid =  Math.floor((Math.random() * 100000));
+                    }
+
+                    // Init job
+                    me.sites[request.path[1]]['jobs'][uid] = {
+                        'uid': uid,
+                        'status': 'pending',
+                        'completed': null,
+                        'commands': requestData
+                    };
+
+                    console.log('Added new job');
+                    console.log(me.sites[request.path[1]]['jobs']);
+
+                    // Emit maintence request with job
+                    me.emit('maintenanceRequested', me.sites[request.path[1]]['jobs'][uid], request.path[1]);
 
                     response.writeHead(200, {'Content-Type':'application/json'});
                     response.end(JSON.stringify({
                         success: true,
-                        message: 'maintenance request finished',
+                        message: 'maintenance request initiated',
+                        job: me.sites[request.path[1]]['jobs'][uid]
                     }));
                     return true;
 
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
index aa72d8f..7a75ff1 100644
--- a/php-bootstrap/maintenance.php
+++ b/php-bootstrap/maintenance.php
@@ -10,18 +10,25 @@
 
 $data = json_decode(file_get_contents('php://input'), true);
 
-foreach ($data['commands'] as $command) {
+foreach ($data['job']['commands'] as &$command) {
     switch ($command['action'])
     {
         case 'cache':
-            handleCacheRequest($command, $data['handle'], $data['siteRoot']);
+            $command['result'] = handleCacheRequest($command, $data['handle'], $data['siteRoot']);
             break;
-        case 'vfs':
-            handleVFSRequest($command, $data['handle'], $data['siteRoot']);
+        case 'vfs-update':
+            $command['result'] = handleVFSUpdateRequest($command, $data['handle'], $data['siteRoot']);
+            break;
+        case 'vfs-summary':
+            $command['result'] = handleVFSSummaryRequest($command, $data['handle'], $data['siteRoot']);
             break;
     }
 }
 
+// Return results
+echo json_encode($data['job']);
+return;
+
 // Create, update or delete cached elements
 function handleCacheRequest($command, $handle, $siteRoot)
 {
@@ -49,13 +56,24 @@ function handleCacheRequest($command, $handle, $siteRoot)
     }
 }
 
-function handleVFSRequest($command, $handle, $siteRoot)
+// Get the vfs summary
+function handleVFSSummaryRequest($command, $handle, $siteRoot)
+{
+    // Initialize site for Emergence FS commands
+    Site::initialize($siteRoot);
+
+    // Update the file system
+    return getFileSystemSummary($command['cursor']);
+}
+
+// Run full update of vfs
+function handleVFSUpdateRequest($command, $handle, $siteRoot)
 {
     // Initialize site for Emergence FS commands
     Site::initialize($siteRoot);
 
     // Update the file system
-    updateFileSystem($handle, $command['cursor'], $siteRoot);
+    return updateFileSystem($handle, $command['cursor'], $siteRoot);
 }
 
 function updateFileSystem($handle, $cursor = 0)

From c23b02c2048c452f963062a2e111304937d09676 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 17:15:06 +0000
Subject: [PATCH 04/43] Add maintence request to sites PATCH

---
 kernel-lib/services/php-fpm.js | 30 ----------------
 kernel-lib/sites.js            | 66 +++++++++++++++++++++++++---------
 php-bootstrap/maintenance.php  |  9 +++--
 3 files changed, 56 insertions(+), 49 deletions(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index f0fc26d..b36a304 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -44,9 +44,6 @@ exports.PhpFpmService = function(name, controller, options) {
         me.status = 'online';
     }
 
-    // listen for site updated
-    controller.sites.on('siteUpdated', _.bind(me.onSiteUpdated, me));
-
     // listen for maintenance requests
     controller.sites.on('maintenanceRequested', _.bind(me.onMaintenanceRequested, me));
 };
@@ -199,33 +196,6 @@ exports.PhpFpmService.prototype.makeConfig = function() {
     return config.join('\n');
 };
 
-exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) {
-    var me = this,
-        siteRoot = me.controller.sites.options.sitesDir + '/' + siteData.handle,
-        phpClient;
-
-    console.log(me.name+': clearing config cache for '+siteRoot);
-
-    // Connect to FPM worker pool
-    phpClient = new phpfpm({
-        sockFile: me.options.socketPath,
-        documentRoot: me.options.bootstrapDir + '/'
-    });
-
-    // Clear cached site.json
-    // @todo update to use maintenance.php instead of cache.php
-    phpClient.run({
-        uri: 'cache.php',
-        json: [
-            { action: 'delete', key: siteRoot }
-        ]
-    }, function(err, output, phpErrors) {
-        if (err == 99) console.error('PHPFPM server error');
-        console.log(output);
-        if (phpErrors) console.error(phpErrors);
-    });
-};
-
 exports.PhpFpmService.prototype.onMaintenanceRequested = function(job, handle) {
     var me = this,
         siteRoot = me.controller.sites.options.sitesDir + '/' + handle,
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index e68ed7d..293a595 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -51,7 +51,6 @@ exports.Sites = function(config) {
 
 util.inherits(exports.Sites, events.EventEmitter);
 
-
 exports.Sites.prototype.handleRequest = function(request, response, server) {
     var me = this;
 
@@ -126,10 +125,41 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             // Restart nginx
             me.emit('siteUpdated', siteData);
 
-            response.writeHead(404, {'Content-Type':'application/json'});
+            // Init / clean jobs
+            if (typeof(me.sites[request.path[1]]['jobs']) == 'undefined') {
+                me.sites[request.path[1]]['jobs'] = {};
+            } else {
+                cleanJobs(me.sites[request.path[1]]['jobs']);
+            }
+
+            // Create uid
+            var uid = Math.floor((Math.random() * 100000));
+            while (Object.keys(me.sites[request.path[1]]['jobs']).indexOf(uid) !== -1) {
+                uid =  Math.floor((Math.random() * 100000));
+            }
+
+            // Init maintenance job
+            me.sites[request.path[1]]['jobs'][uid] = {
+                'uid': uid,
+                'status': 'pending',
+                'completed': null,
+                'commands': [{
+                    'action': 'cache',
+                    'remove': '##SiteRoot##'
+                }]
+            };
+
+            console.log('Added new job');
+            console.log(me.sites[request.path[1]]);
+
+            // Emit maintence request with job
+            me.emit('maintenanceRequested', me.sites[request.path[1]]['jobs'][uid], request.path[1]);
+
+            response.writeHead(200, {'Content-Type':'application/json'});
             response.end(JSON.stringify({
                 success: true,
                 message: 'Processed patch request',
+                job: me.sites[request.path[1]]['jobs'][uid]
             }));
             return;
         }
@@ -227,23 +257,12 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
                     console.log('Received maintenance request for ' + request.path[1]);
                     console.log(requestData);
-                    console.log('Existing jobs');
-                    console.log( me.sites[request.path[1]]['jobs']);
 
-                    // Init jobs array
+                    // Init / clean jobs
                     if (typeof(me.sites[request.path[1]]['jobs']) == 'undefined') {
                         me.sites[request.path[1]]['jobs'] = {};
-                    }
-
-                    // Clean up completed jobs
-                    var jobKeys = Object.keys(me.sites[request.path[1]]['jobs']),
-                        cutoffTime = new Date().getTime() - (60 * 60 * 1000); // 1 hour ago
-                    for (i=0; i<jobKeys.length; i++) {
-                        if (me.sites[request.path[1]]['jobs'][jobKeys[i]].completed && me.sites[request.path[1]]['jobs'][jobKeys[i]].completed < cutoffTime) {
-                            console.log('Remove job');
-                            console.log(me.sites[request.path[1]]['jobs'][jobKeys[i]]);
-                            delete me.sites[request.path[1]]['jobs'][jobKeys[i]];
-                        }
+                    } else {
+                        cleanJobs(me.sites[request.path[1]]['jobs']);
                     }
 
                     // Create uid
@@ -261,7 +280,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     };
 
                     console.log('Added new job');
-                    console.log(me.sites[request.path[1]]['jobs']);
+                    console.log(me.sites[request.path[1]]['jobs'][uid]);
 
                     // Emit maintence request with job
                     me.emit('maintenanceRequested', me.sites[request.path[1]]['jobs'][uid], request.path[1]);
@@ -339,6 +358,19 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
     return false;
 };
 
+// Clean up completed jobs
+cleanJobs = function(jobs) {
+    var jobKeys = Object.keys(jobs),
+        cutoffTime = new Date().getTime() - (60 * 60 * 1000); // 1 hour ago
+
+    for (i=0; i<jobKeys.length; i++) {
+        if (jobs[jobKeys[i]].completed && jobs[jobKeys[i]].completed < cutoffTime) {
+            console.log('Remove job');
+            console.log(jobs[jobKeys[i]]);
+            delete jobs[jobKeys[i]];
+        }
+    }
+}
 
 exports.Sites.prototype.writeSiteConfig = function(requestData) {
     var me = this,
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
index 7a75ff1..4a53027 100644
--- a/php-bootstrap/maintenance.php
+++ b/php-bootstrap/maintenance.php
@@ -33,8 +33,11 @@
 function handleCacheRequest($command, $handle, $siteRoot)
 {
     // Raw delete
-    if (!empty($command['delete'])) {
-        Cache::rawDelete($handle . ':' . $command['delete']);
+    if (!empty($command['remove'])) {
+        if ($command['remove'] == '##SiteRoot##') {
+            $command['remove'] = $siteRoot;
+        }
+        Cache::rawDelete($handle . ':' . $command['remove']);
     }
 
     // Pattern delete
@@ -54,6 +57,8 @@ function handleCacheRequest($command, $handle, $siteRoot)
             empty($command['ttl']) ? 0 : $command['ttl']
         );
     }
+
+    return true;
 }
 
 // Get the vfs summary

From 73febfee583cdb19b16a51b8a5ccd02248ccb9ef Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 17:15:32 +0000
Subject: [PATCH 05/43] Add jobs getter to site request

---
 kernel-lib/sites.js | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 293a595..bb00401 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -68,12 +68,25 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                 if (request.path[2] == 'maintenance') {
                     console.log('Received maintenance GET request for ' + request.path[1]);
                     response.writeHead(200, {'Content-Type':'application/json'});
-                    response.end(JSON.stringify({
-                        success: true,
-                        message: 'maintenance request finished',
-                        jobs: me.sites[request.path[1]].jobs
-                    }));
-                    return true;
+
+                    // Return specific uid
+                    if (request.path[3]) {
+                        response.end(JSON.stringify({
+                            success: true,
+                            message: 'Maintenance get request finished',
+                            jobs: (me.sites[request.path[1]].jobs[request.path[3]]) ? me.sites[request.path[1]].jobs[request.path[3]] : false
+                        }));
+                        return true;
+
+                    // Return all jobs
+                    } else {
+                        response.end(JSON.stringify({
+                            success: true,
+                            message: 'Maintenance get request finished',
+                            jobs: (me.sites[request.path[1]].jobs) ? me.sites[request.path[1]].jobs : false
+                        }));
+                        return true;
+                    }
 
                 } else {
                     console.error('Unhandled site sub-resource: ' + request.path[2]);

From 73b2bca92b8a6e959a754d24564af0970fe2df40 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 18:03:22 +0000
Subject: [PATCH 06/43] Fix bug which caused jobs to get written to the disk

---
 kernel-lib/sites.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index bb00401..e026a47 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -120,7 +120,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                 siteDir = me.options.sitesDir + '/' + handle,
                 siteConfigPath = siteDir + '/site.json',
                 params = JSON.parse(request.content),
-                siteData;
+                siteData, siteDataTmp;
 
             // Get existing site config
             siteData = me.sites[handle];
@@ -132,8 +132,10 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                 }
             }
 
-            // Update file
-            fs.writeFileSync(siteConfigPath, JSON.stringify(siteData, null, 4));
+            // Clone site data and remove jobs before writing sitedata to disk
+            siteDataTmp = JSON.parse(JSON.stringify(siteData));;
+            delete siteDataTmp.jobs;
+            fs.writeFileSync(siteConfigPath, JSON.stringify(siteDataTmp, null, 4));
 
             // Restart nginx
             me.emit('siteUpdated', siteData);

From a25d1f5faf4dca3784053b42c97661c36ae81a6c Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 18:04:29 +0000
Subject: [PATCH 07/43] Remove site key from site root cache deletion

---
 php-bootstrap/maintenance.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
index 4a53027..df160da 100644
--- a/php-bootstrap/maintenance.php
+++ b/php-bootstrap/maintenance.php
@@ -11,6 +11,7 @@
 $data = json_decode(file_get_contents('php://input'), true);
 
 foreach ($data['job']['commands'] as &$command) {
+
     switch ($command['action'])
     {
         case 'cache':
@@ -35,9 +36,10 @@ function handleCacheRequest($command, $handle, $siteRoot)
     // Raw delete
     if (!empty($command['remove'])) {
         if ($command['remove'] == '##SiteRoot##') {
-            $command['remove'] = $siteRoot;
+            Cache::rawDelete($siteRoot);
+        } else {
+            Cache::rawDelete($handle . ':' . $command['remove']);
         }
-        Cache::rawDelete($handle . ':' . $command['remove']);
     }
 
     // Pattern delete

From a273c204c36d825f9439eb91bbb3820723cd9484 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 18:05:10 +0000
Subject: [PATCH 08/43] Remove deprecated cache.php file

---
 php-bootstrap/cache.php | 29 -----------------------------
 1 file changed, 29 deletions(-)
 delete mode 100644 php-bootstrap/cache.php

diff --git a/php-bootstrap/cache.php b/php-bootstrap/cache.php
deleted file mode 100644
index 708b812..0000000
--- a/php-bootstrap/cache.php
+++ /dev/null
@@ -1,29 +0,0 @@
-<?php
-
-// bootstrap emergence
-require('bootstrap.inc.php');
-
-// only process POST requests
-if ($_SERVER['REQUEST_METHOD'] != 'POST') {
-    exit();
-}
-
-$commands = json_decode(file_get_contents('php://input'), true);
-
-foreach ($commands AS $command) {
-    $key = $command['key'];
-
-    if (!empty($command['site'])) {
-        $key = $command['site'].':'.$key;
-    }
-
-    if ($action == 'delete') {
-        Cache::rawDelete($key);
-    } else {
-        Cache::rawStore(
-            $key,
-            empty($command['value']) ? null : $command['value'],
-            empty($command['ttl']) ? 0 : $command['ttl']
-        );
-    }
-}
\ No newline at end of file

From f72ecb3535fdee3248fd6f412650ebe4239b9bb7 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 11 May 2017 18:10:02 +0000
Subject: [PATCH 09/43] Remove unnecessary phpfpm param

---
 kernel-lib/services/php-fpm.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index b36a304..5702326 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -203,7 +203,6 @@ exports.PhpFpmService.prototype.onMaintenanceRequested = function(job, handle) {
 
     // Connect to FPM worker pool
     phpClient = new phpfpm({
-        handle: handle,
         sockFile: me.options.socketPath,
         documentRoot: me.options.bootstrapDir + '/'
     });

From 694a198b6716d653cd54c0cd331bd7f110309df3 Mon Sep 17 00:00:00 2001
From: Chris Alfano <chris@jarv.us>
Date: Fri, 12 May 2017 12:56:03 -0400
Subject: [PATCH 10/43] Store site object in variable

---
 kernel-lib/sites.js | 54 +++++++++++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index e026a47..7f89dca 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -52,12 +52,15 @@ exports.Sites = function(config) {
 util.inherits(exports.Sites, events.EventEmitter);
 
 exports.Sites.prototype.handleRequest = function(request, response, server) {
-    var me = this;
+    var me = this,
+        site;
 
     if (request.method == 'GET') {
 
         if (request.path[1]) {
-            if (!me.sites[request.path[1]]) {
+            site = me.sites[request.path[1]];
+
+            if (!site) {
                 console.error('Site not found: ' + request.path[1]);
                 response.writeHead(404, {'Content-Type':'application/json'});
                 response.end(JSON.stringify({success: false, message: 'Site not found'}));
@@ -74,7 +77,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                         response.end(JSON.stringify({
                             success: true,
                             message: 'Maintenance get request finished',
-                            jobs: (me.sites[request.path[1]].jobs[request.path[3]]) ? me.sites[request.path[1]].jobs[request.path[3]] : false
+                            jobs: (site.jobs[request.path[3]]) ? site.jobs[request.path[3]] : false
                         }));
                         return true;
 
@@ -83,7 +86,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                         response.end(JSON.stringify({
                             success: true,
                             message: 'Maintenance get request finished',
-                            jobs: (me.sites[request.path[1]].jobs) ? me.sites[request.path[1]].jobs : false
+                            jobs: (site.jobs) ? site.jobs : false
                         }));
                         return true;
                     }
@@ -97,7 +100,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             }
 
             response.writeHead(200, {'Content-Type':'application/json'});
-            response.end(JSON.stringify({data: me.sites[request.path[1]]}));
+            response.end(JSON.stringify({data: site}));
             return true;
         }
 
@@ -108,8 +111,9 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
     } else if (request.method == 'PATCH') {
 
         if (request.path[1]) {
+            site = me.sites[request.path[1]];
 
-            if (!me.sites[request.path[1]]) {
+            if (!site) {
                 console.error('Site not found: ' + request.path[1]);
                 response.writeHead(404, {'Content-Type':'application/json'});
                 response.end(JSON.stringify({success: false, message: 'Site not found'}));
@@ -141,20 +145,20 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             me.emit('siteUpdated', siteData);
 
             // Init / clean jobs
-            if (typeof(me.sites[request.path[1]]['jobs']) == 'undefined') {
-                me.sites[request.path[1]]['jobs'] = {};
+            if (typeof(site['jobs']) == 'undefined') {
+                site['jobs'] = {};
             } else {
-                cleanJobs(me.sites[request.path[1]]['jobs']);
+                cleanJobs(site['jobs']);
             }
 
             // Create uid
             var uid = Math.floor((Math.random() * 100000));
-            while (Object.keys(me.sites[request.path[1]]['jobs']).indexOf(uid) !== -1) {
+            while (Object.keys(site['jobs']).indexOf(uid) !== -1) {
                 uid =  Math.floor((Math.random() * 100000));
             }
 
             // Init maintenance job
-            me.sites[request.path[1]]['jobs'][uid] = {
+            site['jobs'][uid] = {
                 'uid': uid,
                 'status': 'pending',
                 'completed': null,
@@ -165,16 +169,16 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             };
 
             console.log('Added new job');
-            console.log(me.sites[request.path[1]]);
+            console.log(site);
 
             // Emit maintence request with job
-            me.emit('maintenanceRequested', me.sites[request.path[1]]['jobs'][uid], request.path[1]);
+            me.emit('maintenanceRequested', site['jobs'][uid], request.path[1]);
 
             response.writeHead(200, {'Content-Type':'application/json'});
             response.end(JSON.stringify({
                 success: true,
                 message: 'Processed patch request',
-                job: me.sites[request.path[1]]['jobs'][uid]
+                job: site['jobs'][uid]
             }));
             return;
         }
@@ -195,7 +199,9 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
         // handle post to an individual site
         if (request.path[1]) {
-            if (!me.sites[request.path[1]]) {
+            site = me.sites[request.path[1]];
+
+            if (!site) {
                 console.error('Site not found: ' + request.path[1]);
                 response.writeHead(404, {'Content-Type':'application/json'});
                 response.end(JSON.stringify({success: false, message: 'Site not found'}));
@@ -274,20 +280,20 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     console.log(requestData);
 
                     // Init / clean jobs
-                    if (typeof(me.sites[request.path[1]]['jobs']) == 'undefined') {
-                        me.sites[request.path[1]]['jobs'] = {};
+                    if (typeof(site['jobs']) == 'undefined') {
+                        site['jobs'] = {};
                     } else {
-                        cleanJobs(me.sites[request.path[1]]['jobs']);
+                        cleanJobs(site['jobs']);
                     }
 
                     // Create uid
                     var uid = Math.floor((Math.random() * 100000));
-                    while (Object.keys(me.sites[request.path[1]]['jobs']).indexOf(uid) !== -1) {
+                    while (Object.keys(site['jobs']).indexOf(uid) !== -1) {
                         uid =  Math.floor((Math.random() * 100000));
                     }
 
                     // Init job
-                    me.sites[request.path[1]]['jobs'][uid] = {
+                    site['jobs'][uid] = {
                         'uid': uid,
                         'status': 'pending',
                         'completed': null,
@@ -295,16 +301,16 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     };
 
                     console.log('Added new job');
-                    console.log(me.sites[request.path[1]]['jobs'][uid]);
+                    console.log(site['jobs'][uid]);
 
                     // Emit maintence request with job
-                    me.emit('maintenanceRequested', me.sites[request.path[1]]['jobs'][uid], request.path[1]);
+                    me.emit('maintenanceRequested', site['jobs'][uid], request.path[1]);
 
                     response.writeHead(200, {'Content-Type':'application/json'});
                     response.end(JSON.stringify({
                         success: true,
                         message: 'maintenance request initiated',
-                        job: me.sites[request.path[1]]['jobs'][uid]
+                        job: site['jobs'][uid]
                     }));
                     return true;
 
@@ -317,7 +323,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             }
 
             // apply existing site's properties in
-            _.defaults(requestData, me.sites[request.path[1]]);
+            _.defaults(requestData, site);
         }
 
         // create new site

From 99edfa538c25c86f401b9f0db272d2b76c911286 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 15 May 2017 19:06:42 +0000
Subject: [PATCH 11/43] Remove extra semicolon

---
 kernel-lib/sites.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 7f89dca..e1118fa 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -137,7 +137,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             }
 
             // Clone site data and remove jobs before writing sitedata to disk
-            siteDataTmp = JSON.parse(JSON.stringify(siteData));;
+            siteDataTmp = JSON.parse(JSON.stringify(siteData));
             delete siteDataTmp.jobs;
             fs.writeFileSync(siteConfigPath, JSON.stringify(siteDataTmp, null, 4));
 

From 505c2591de3470c1d736a6aff1cc1ae9a33e3cf8 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Tue, 16 May 2017 23:06:08 +0000
Subject: [PATCH 12/43] Add config wrapper around siteData

---
 kernel-lib/services/nginx.js | 18 +++++++++---------
 kernel-lib/sites.js          | 26 +++++++++++++++++---------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/kernel-lib/services/nginx.js b/kernel-lib/services/nginx.js
index b42aba3..41a2fb2 100644
--- a/kernel-lib/services/nginx.js
+++ b/kernel-lib/services/nginx.js
@@ -256,15 +256,15 @@ exports.NginxService.prototype.makeConfig = function() {
     );
 
     _.each(me.controller.sites.sites, function(site, handle) {
-        var hostnames = site.hostnames.slice(),
+        var hostnames = site.config.hostnames.slice(),
             siteDir = me.controller.sites.options.sitesDir+'/'+handle,
             logsDir = siteDir+'/logs',
             siteConfig = [],
             sslHostnames, sslHostname;
 
         // process hostnames
-        if (_.indexOf(hostnames, site.primary_hostname) == -1) {
-            hostnames.unshift(site.primary_hostname);
+        if (_.indexOf(hostnames, site.config.primary_hostname) == -1) {
+            hostnames.unshift(site.config.primary_hostname);
         }
 
         // process directories
@@ -301,15 +301,15 @@ exports.NginxService.prototype.makeConfig = function() {
             '    }'
         );
 
-        if (site.ssl) {
-            if (site.ssl.hostnames) {
-                sslHostnames = site.ssl.hostnames;
+        if (site.config.ssl) {
+            if (site.config.ssl.hostnames) {
+                sslHostnames = site.config.ssl.hostnames;
             } else {
                 sslHostnames = {};
-                sslHostnames[site.primary_hostname] = site.ssl;
+                sslHostnames[site.config.primary_hostname] = site.config.ssl;
 
-                site.hostnames.forEach(function(hostname) {
-                    sslHostnames[hostname] = site.ssl;
+                site.config.hostnames.forEach(function(hostname) {
+                    sslHostnames[hostname] = site.config.ssl;
                 });
             }
 
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index e1118fa..ecaa5b7 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -39,8 +39,11 @@ exports.Sites = function(config) {
     me.sites = {};
     _.each(fs.readdirSync(me.options.sitesDir), function(handle) {
         try {
-            me.sites[handle] = JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii'));
-            me.sites[handle].handle = handle;
+            me.sites[handle] = {
+                handle: handle,
+                config: JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii')),
+                jobs: []
+            };
             console.log('-Loaded: '+me.sites[handle].primary_hostname);
         } catch (error) {
             console.log('-FAILED to load: '+handle);
@@ -100,12 +103,12 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             }
 
             response.writeHead(200, {'Content-Type':'application/json'});
-            response.end(JSON.stringify({data: site}));
+            response.end(JSON.stringify({data: site.config}));
             return true;
         }
 
         response.writeHead(200, {'Content-Type':'application/json'});
-        response.end(JSON.stringify({data: _.values(me.sites)}));
+        response.end(JSON.stringify({data: _.pluck(_.values(me.sites), 'config')}));
         return true;
 
     } else if (request.method == 'PATCH') {
@@ -328,6 +331,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
         // create new site
         try {
+
             cfgResult = me.writeSiteConfig(requestData);
 
             if (cfgResult.isNew) {
@@ -452,7 +456,11 @@ exports.Sites.prototype.writeSiteConfig = function(requestData) {
     }
 
     // write site config to file
-    this.sites[siteData.handle] = siteData;
+    this.sites[siteData.handle] = {
+        handle: siteData.handle,
+        config: siteData,
+        jobs: []
+    };
 
     var isNew = !fs.existsSync(siteConfigPath);
 
@@ -470,12 +478,12 @@ exports.Sites.prototype.updateSiteConfig = function(handle, changes) {
         siteData = this.sites[handle],
         create_user;
 
-    _.extend(siteData, changes);
+    _.extend(siteData.config, changes);
 
-    create_user = siteData.create_user;
-    delete siteData.create_user;
+    create_user = siteData.config.create_user;
+    delete siteData.config.create_user;
 
-    fs.writeFileSync(filename, JSON.stringify(this.sites[handle], null, 4));
+    fs.writeFileSync(filename, JSON.stringify(this.sites[handle].config, null, 4));
 
     if (create_user) {
         siteData.create_user = create_user;

From d76c5151ee29d1df339197ff25ed12e4c3ac518b Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Tue, 16 May 2017 23:09:54 +0000
Subject: [PATCH 13/43] Clean up jobs references

---
 kernel-lib/sites.js | 32 +++++++++-----------------------
 1 file changed, 9 insertions(+), 23 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index ecaa5b7..b931438 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -147,21 +147,14 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             // Restart nginx
             me.emit('siteUpdated', siteData);
 
-            // Init / clean jobs
-            if (typeof(site['jobs']) == 'undefined') {
-                site['jobs'] = {};
-            } else {
-                cleanJobs(site['jobs']);
-            }
-
             // Create uid
             var uid = Math.floor((Math.random() * 100000));
-            while (Object.keys(site['jobs']).indexOf(uid) !== -1) {
+            while (Object.keys(site.jobs).indexOf(uid) !== -1) {
                 uid =  Math.floor((Math.random() * 100000));
             }
 
             // Init maintenance job
-            site['jobs'][uid] = {
+            site.jobs[uid] = {
                 'uid': uid,
                 'status': 'pending',
                 'completed': null,
@@ -175,13 +168,13 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             console.log(site);
 
             // Emit maintence request with job
-            me.emit('maintenanceRequested', site['jobs'][uid], request.path[1]);
+            me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
 
             response.writeHead(200, {'Content-Type':'application/json'});
             response.end(JSON.stringify({
                 success: true,
                 message: 'Processed patch request',
-                job: site['jobs'][uid]
+                job: site.jobs[uid]
             }));
             return;
         }
@@ -282,21 +275,14 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     console.log('Received maintenance request for ' + request.path[1]);
                     console.log(requestData);
 
-                    // Init / clean jobs
-                    if (typeof(site['jobs']) == 'undefined') {
-                        site['jobs'] = {};
-                    } else {
-                        cleanJobs(site['jobs']);
-                    }
-
                     // Create uid
                     var uid = Math.floor((Math.random() * 100000));
-                    while (Object.keys(site['jobs']).indexOf(uid) !== -1) {
+                    while (Object.keys(site.jobs).indexOf(uid) !== -1) {
                         uid =  Math.floor((Math.random() * 100000));
                     }
 
                     // Init job
-                    site['jobs'][uid] = {
+                    site.jobs[uid] = {
                         'uid': uid,
                         'status': 'pending',
                         'completed': null,
@@ -304,16 +290,16 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     };
 
                     console.log('Added new job');
-                    console.log(site['jobs'][uid]);
+                    console.log(site.jobs[uid]);
 
                     // Emit maintence request with job
-                    me.emit('maintenanceRequested', site['jobs'][uid], request.path[1]);
+                    me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
 
                     response.writeHead(200, {'Content-Type':'application/json'});
                     response.end(JSON.stringify({
                         success: true,
                         message: 'maintenance request initiated',
-                        job: site['jobs'][uid]
+                        job: site.jobs[uid]
                     }));
                     return true;
 

From 7e9a20b9ce0c9d9a131b373e73f96b8eb706da13 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 00:18:04 +0000
Subject: [PATCH 14/43] Add npm package uuid

---
 package.json | 55 ++++++++++++++++++++++++++--------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)

diff --git a/package.json b/package.json
index 01ebf42..bfc2231 100644
--- a/package.json
+++ b/package.json
@@ -1,29 +1,30 @@
 {
-    "name": "emergence",
-    "preferGlobal": true,
-    "version": "0.0.3",
-    "license": "MIT",
-    "dependencies": {
-        "underscore": "1.3.x",
-        "mariasql": "^0.2.6",
-        "node-static": "0.6.x",
-        "http-auth": "1.2.7",
-        "posix": "^4.0.2",
-        "hostile": "^1.0.2",
-        "shelljs": "^0.5.3",
-        "semver": "^5.1.0",
-        "underscore-cli": "^0.2.19",
-        "node-phpfpm": "^1.0.1"
-    },
-    "bin": {
-        "emergence-kernel": "./bin/kernel",
-        "emergence-shell": "./bin/shell",
-        "emergence-git-shell": "./bin/git-shell",
-        "emergence-su": "./bin/su",
-        "emergence-create-site": "./bin/create-site.sh"
-    },
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/JarvusInnovations/Emergence.git"
-    }
+  "name": "emergence",
+  "preferGlobal": true,
+  "version": "0.0.3",
+  "license": "MIT",
+  "dependencies": {
+    "underscore": "1.3.x",
+    "mariasql": "^0.2.6",
+    "node-static": "0.6.x",
+    "http-auth": "1.2.7",
+    "posix": "^4.0.2",
+    "hostile": "^1.0.2",
+    "shelljs": "^0.5.3",
+    "semver": "^5.1.0",
+    "underscore-cli": "^0.2.19",
+    "node-phpfpm": "^1.0.1",
+    "uuid": "~3.0.1"
+  },
+  "bin": {
+    "emergence-kernel": "./bin/kernel",
+    "emergence-shell": "./bin/shell",
+    "emergence-git-shell": "./bin/git-shell",
+    "emergence-su": "./bin/su",
+    "emergence-create-site": "./bin/create-site.sh"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/JarvusInnovations/Emergence.git"
+  }
 }

From 456e4938ca663533fa833d5985c49f1d8ef63844 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 00:19:03 +0000
Subject: [PATCH 15/43] Init job as object not array

---
 kernel-lib/sites.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index b931438..3757bd7 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -42,7 +42,7 @@ exports.Sites = function(config) {
             me.sites[handle] = {
                 handle: handle,
                 config: JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii')),
-                jobs: []
+                jobs: {}
             };
             console.log('-Loaded: '+me.sites[handle].primary_hostname);
         } catch (error) {

From 938a036af3baec4ae0b36b6698c87a483b03f02f Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 00:20:20 +0000
Subject: [PATCH 16/43] Use uuid package over hand crafted uuids

---
 kernel-lib/sites.js | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 3757bd7..bf07895 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -7,7 +7,8 @@ var _ = require('underscore'),
     posix = require('posix'),
     spawn = require('child_process').spawn,
     hostile = require('hostile'),
-    phpShellScript = path.resolve(__dirname, '../bin/shell');
+    phpShellScript = path.resolve(__dirname, '../bin/shell'),
+    uuidV1 = require('uuid/v1');
 
 
 exports.createSites = function(config) {
@@ -127,7 +128,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                 siteDir = me.options.sitesDir + '/' + handle,
                 siteConfigPath = siteDir + '/site.json',
                 params = JSON.parse(request.content),
-                siteData, siteDataTmp;
+                siteData, siteDataTmp, uid;
 
             // Get existing site config
             siteData = me.sites[handle];
@@ -148,10 +149,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             me.emit('siteUpdated', siteData);
 
             // Create uid
-            var uid = Math.floor((Math.random() * 100000));
-            while (Object.keys(site.jobs).indexOf(uid) !== -1) {
-                uid =  Math.floor((Math.random() * 100000));
-            }
+            uid = uuidV1();
 
             // Init maintenance job
             site.jobs[uid] = {
@@ -276,10 +274,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     console.log(requestData);
 
                     // Create uid
-                    var uid = Math.floor((Math.random() * 100000));
-                    while (Object.keys(site.jobs).indexOf(uid) !== -1) {
-                        uid =  Math.floor((Math.random() * 100000));
-                    }
+                    var uid = uuidV1();
 
                     // Init job
                     site.jobs[uid] = {

From b999d4b443c2a975072709ee565a97fe853605ea Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 00:21:04 +0000
Subject: [PATCH 17/43] Restructure job init

---
 kernel-lib/sites.js | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index bf07895..846f6ba 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -136,14 +136,12 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             // Apply updates except handle
             for (var k in params) {
                 if (k !== 'handle') {
-                    siteData[k] = params[k];
+                    siteData.config[k] = params[k];
                 }
             }
 
-            // Clone site data and remove jobs before writing sitedata to disk
-            siteDataTmp = JSON.parse(JSON.stringify(siteData));
-            delete siteDataTmp.jobs;
-            fs.writeFileSync(siteConfigPath, JSON.stringify(siteDataTmp, null, 4));
+            // Update config file
+            fs.writeFileSync(siteConfigPath, JSON.stringify(siteData.config, null, 4));
 
             // Restart nginx
             me.emit('siteUpdated', siteData);
@@ -155,11 +153,13 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             site.jobs[uid] = {
                 'uid': uid,
                 'status': 'pending',
+                'received': new Date().getTime(),
+                'started': null,
                 'completed': null,
-                'commands': [{
+                'command': {
                     'action': 'cache',
                     'remove': '##SiteRoot##'
-                }]
+                }
             };
 
             console.log('Added new job');
@@ -280,8 +280,10 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     site.jobs[uid] = {
                         'uid': uid,
                         'status': 'pending',
+                        'received': new Date().getTime(),
+                        'started': null,
                         'completed': null,
-                        'commands': requestData
+                        'command': requestData
                     };
 
                     console.log('Added new job');

From 24018ba4d1de3b07d9ddc84f74e023b6c108fc39 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 00:27:39 +0000
Subject: [PATCH 18/43] Restructure pruneJobs

---
 kernel-lib/sites.js | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 846f6ba..4f2c3bb 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -133,6 +133,9 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             // Get existing site config
             siteData = me.sites[handle];
 
+            // Prune jobs
+            pruneJobs(siteData.jobs);
+
             // Apply updates except handle
             for (var k in params) {
                 if (k !== 'handle') {
@@ -273,6 +276,9 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     console.log('Received maintenance request for ' + request.path[1]);
                     console.log(requestData);
 
+                    // Prune jobs
+                    pruneJobs(site.jobs);
+
                     // Create uid
                     var uid = uuidV1();
 
@@ -367,13 +373,17 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 };
 
 // Clean up completed jobs
-cleanJobs = function(jobs) {
+function pruneJobs(jobs) {
     var jobKeys = Object.keys(jobs),
         cutoffTime = new Date().getTime() - (60 * 60 * 1000); // 1 hour ago
 
     for (i=0; i<jobKeys.length; i++) {
         if (jobs[jobKeys[i]].completed && jobs[jobKeys[i]].completed < cutoffTime) {
-            console.log('Remove job');
+            console.log('Remove completed job');
+            console.log(jobs[jobKeys[i]]);
+            delete jobs[jobKeys[i]];
+        } else if (jobs[jobKeys[i]].status == 'failed' && jobs[jobKeys[i]].received < cutoffTime) {
+            console.log('Remove failed job');
             console.log(jobs[jobKeys[i]]);
             delete jobs[jobKeys[i]];
         }

From baae2281eb0d806af536acbe265c5b60212ad4a2 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 19:58:28 +0000
Subject: [PATCH 19/43] Restructure jobs to give eacn command a unique uid

---
 kernel-lib/sites.js | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 4f2c3bb..969cf84 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -166,7 +166,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             };
 
             console.log('Added new job');
-            console.log(site);
+            console.log(site.jobs);
 
             // Emit maintence request with job
             me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
@@ -276,33 +276,40 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     console.log('Received maintenance request for ' + request.path[1]);
                     console.log(requestData);
 
+                    var uid, newJobs = [];
+
                     // Prune jobs
                     pruneJobs(site.jobs);
 
-                    // Create uid
-                    var uid = uuidV1();
+                    for (i=0; i<requestData.length; i++) {
+
+                        // Create uid
+                        uid = uuidV1();
 
-                    // Init job
-                    site.jobs[uid] = {
-                        'uid': uid,
-                        'status': 'pending',
-                        'received': new Date().getTime(),
-                        'started': null,
-                        'completed': null,
-                        'command': requestData
-                    };
+                        // Init job
+                        site.jobs[uid] = {
+                            'uid': uid,
+                            'status': 'pending',
+                            'received': new Date().getTime(),
+                            'started': null,
+                            'completed': null,
+                            'command': requestData[i]
+                        };
 
-                    console.log('Added new job');
-                    console.log(site.jobs[uid]);
+                        // Append new job
+                        newJobs.push(site.jobs[uid]);
+                        console.log('Added new job');
+                        console.log(site.jobs[uid]);
 
-                    // Emit maintence request with job
-                    me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
+                        // Emit maintence request with job
+                        me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
+                    }
 
                     response.writeHead(200, {'Content-Type':'application/json'});
                     response.end(JSON.stringify({
                         success: true,
                         message: 'maintenance request initiated',
-                        job: site.jobs[uid]
+                        jobs: newJobs
                     }));
                     return true;
 

From dad9f789460aae5dd2f7ec980da02623e0b9ceee Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 20:00:16 +0000
Subject: [PATCH 20/43] Add async npm package Used command:     npm install
 async --save

---
 package.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index bfc2231..a918715 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,8 @@
     "semver": "^5.1.0",
     "underscore-cli": "^0.2.19",
     "node-phpfpm": "^1.0.1",
-    "uuid": "~3.0.1"
+    "uuid": "~3.0.1",
+    "async": "~2.4.0"
   },
   "bin": {
     "emergence-kernel": "./bin/kernel",

From 099e07921c01ad357c2dbdc65d90d784392bcf23 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 20:31:28 +0000
Subject: [PATCH 21/43] Add async maintenance queue

---
 kernel-lib/services/php-fpm.js | 45 +++++++++++++++++++++++++---------
 kernel-lib/sites.js            |  4 ++-
 php-bootstrap/maintenance.php  | 25 +++++++++----------
 3 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 5702326..1feef83 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -3,7 +3,9 @@ var _ = require('underscore'),
     path = require('path'),
     util = require('util'),
     spawn = require('child_process').spawn,
-    phpfpm = require('node-phpfpm');
+    phpfpm = require('node-phpfpm'),
+    async = require('async'),
+    maintenenceQueue;
 
 exports.createService = function(name, controller, options) {
     return new exports.PhpFpmService(name, controller, options);
@@ -196,9 +198,9 @@ exports.PhpFpmService.prototype.makeConfig = function() {
     return config.join('\n');
 };
 
-exports.PhpFpmService.prototype.onMaintenanceRequested = function(job, handle) {
-    var me = this,
-        siteRoot = me.controller.sites.options.sitesDir + '/' + handle,
+maintenenceQueue = async.queue(function(data, callback) {
+        var me = data.scope,
+        siteRoot = me.controller.sites.options.sitesDir + '/' + data.job.handle,
         phpClient;
 
     // Connect to FPM worker pool
@@ -207,24 +209,43 @@ exports.PhpFpmService.prototype.onMaintenanceRequested = function(job, handle) {
         documentRoot: me.options.bootstrapDir + '/'
     });
 
+    // Mark job as started
+    data.job.started = new Date().getTime();
+
     // Run maintenance request
     phpClient.run({
         uri: 'maintenance.php',
         json: {
-            'job': job,
-            'handle': handle,
+            'job': data.job,
+            'handle': data.job.handle,
             'siteRoot': siteRoot
         }
-    }, function(err, output, phpErrors) {
-        if (err == 99) console.error('PHPFPM server error');
-        if (phpErrors) console.error(phpErrors);
+    }, function(err, output, stderr) {
+        if (err == 99) {
+            data.job.status = 'failed';
+            data.job.message = 'PHPFPM server error';
+            console.error(data.job.message);
+            return callback(err);
+        }
+        if (stderr) {
+            data.job.status = 'failed';
+            data.job.message = stderr;
+            console.error(stderr);
+            return callback(stderr);
+        }
 
         // Parse job response
         var response = JSON.parse(output);
 
         // Update job with response
-        job.commands = response.commands;
-        job.status = 'completed';
-        job.completed = new Date().getTime();
+        data.job.command = response.command;
+        data.job.status = 'completed';
+        data.job.completed = new Date().getTime();
+
+        callback(null, data.job);
     });
+}, 5);
+
+exports.PhpFpmService.prototype.onMaintenanceRequested = function(job) {
+    maintenenceQueue.push({'job': job, 'scope': this});
 }
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 969cf84..f2b8a67 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -155,6 +155,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             // Init maintenance job
             site.jobs[uid] = {
                 'uid': uid,
+                'handle': request.path[1],
                 'status': 'pending',
                 'received': new Date().getTime(),
                 'started': null,
@@ -289,11 +290,12 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                         // Init job
                         site.jobs[uid] = {
                             'uid': uid,
+                            'handle': request.path[1],
                             'status': 'pending',
                             'received': new Date().getTime(),
                             'started': null,
                             'completed': null,
-                            'command': requestData[i]
+                            'command': requestData[i],
                         };
 
                         // Append new job
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
index df160da..8d39424 100644
--- a/php-bootstrap/maintenance.php
+++ b/php-bootstrap/maintenance.php
@@ -10,20 +10,17 @@
 
 $data = json_decode(file_get_contents('php://input'), true);
 
-foreach ($data['job']['commands'] as &$command) {
-
-    switch ($command['action'])
-    {
-        case 'cache':
-            $command['result'] = handleCacheRequest($command, $data['handle'], $data['siteRoot']);
-            break;
-        case 'vfs-update':
-            $command['result'] = handleVFSUpdateRequest($command, $data['handle'], $data['siteRoot']);
-            break;
-        case 'vfs-summary':
-            $command['result'] = handleVFSSummaryRequest($command, $data['handle'], $data['siteRoot']);
-            break;
-    }
+switch ($data['job']['command']['action'])
+{
+    case 'cache':
+        $data['job']['command']['result'] = handleCacheRequest($command, $data['handle'], $data['siteRoot']);
+        break;
+    case 'vfs-update':
+        $data['job']['command']['result'] = handleVFSUpdateRequest($command, $data['handle'], $data['siteRoot']);
+        break;
+    case 'vfs-summary':
+        $data['job']['command']['result'] = handleVFSSummaryRequest($command, $data['handle'], $data['siteRoot']);
+        break;
 }
 
 // Return results

From dd154d10363f5a304f3f006a6fbb2e9e73f2a701 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 20:37:54 +0000
Subject: [PATCH 22/43] Add config reload maintenance action

---
 kernel-lib/sites.js           |  3 +--
 php-bootstrap/maintenance.php | 16 +++++++++++-----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index f2b8a67..2646947 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -161,8 +161,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                 'started': null,
                 'completed': null,
                 'command': {
-                    'action': 'cache',
-                    'remove': '##SiteRoot##'
+                    'action': 'config-reload'
                 }
             };
 
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/maintenance.php
index 8d39424..f77765b 100644
--- a/php-bootstrap/maintenance.php
+++ b/php-bootstrap/maintenance.php
@@ -15,6 +15,9 @@
     case 'cache':
         $data['job']['command']['result'] = handleCacheRequest($command, $data['handle'], $data['siteRoot']);
         break;
+    case 'config-reload':
+        $data['job']['command']['result'] = handleConfigReload($command, $data['handle'], $data['siteRoot']);
+        break;
     case 'vfs-update':
         $data['job']['command']['result'] = handleVFSUpdateRequest($command, $data['handle'], $data['siteRoot']);
         break;
@@ -32,11 +35,7 @@ function handleCacheRequest($command, $handle, $siteRoot)
 {
     // Raw delete
     if (!empty($command['remove'])) {
-        if ($command['remove'] == '##SiteRoot##') {
-            Cache::rawDelete($siteRoot);
-        } else {
-            Cache::rawDelete($handle . ':' . $command['remove']);
-        }
+        Cache::rawDelete($handle . ':' . $command['remove']);
     }
 
     // Pattern delete
@@ -60,6 +59,13 @@ function handleCacheRequest($command, $handle, $siteRoot)
     return true;
 }
 
+// Remove the site config cache
+function handleConfigReload($command, $handle, $siteRoot)
+{
+    Cache::rawDelete($siteRoot);
+    return true;
+}
+
 // Get the vfs summary
 function handleVFSSummaryRequest($command, $handle, $siteRoot)
 {

From ddce23388249487622180db388a1b272a94ec657 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Wed, 17 May 2017 21:14:00 +0000
Subject: [PATCH 23/43] Add bulk site maintenance request handler

---
 kernel-lib/sites.js | 59 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 3 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 2646947..f2a377b 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -168,7 +168,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             console.log('Added new job');
             console.log(site.jobs);
 
-            // Emit maintence request with job
+            // Emit maintenance request with job
             me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
 
             response.writeHead(200, {'Content-Type':'application/json'});
@@ -194,8 +194,60 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             requestData = request.content;
         }
 
+        // handle bulk maintenance request
+        if (request.path[1] == 'maintenance') {
+
+            console.log('Received bulk maintenance request');
+            console.log(requestData);
+
+            var uid, newJobs = [];
+
+            for (a=0; a<requestData.length; a++) {
+
+                // Find site by handle
+                site = me.sites[requestData[a].handle];
+
+                if (!site) {
+                    console.error('Site not found: ' + me.sites[requestData[a].handle]);
+                    continue;
+                }
+
+                // Prune jobs
+                pruneJobs(site.jobs);
+
+                // Create uid
+                uid = uuidV1();
+
+                // Init job
+                site.jobs[uid] = {
+                    'uid': uid,
+                    'handle': requestData[a].handle,
+                    'status': 'pending',
+                    'received': new Date().getTime(),
+                    'started': null,
+                    'completed': null,
+                    'command': requestData[a],
+                };
+
+                // Append new job
+                newJobs.push(site.jobs[uid]);
+                console.log('Added new job');
+                console.log(site.jobs[uid]);
+
+                // Emit maintenance request with job
+                me.emit('maintenanceRequested', site.jobs[uid], requestData[a].handle);
+            }
+
+            response.writeHead(200, {'Content-Type':'application/json'});
+            response.end(JSON.stringify({
+                success: true,
+                message: 'maintenance request initiated',
+                jobs: newJobs
+            }));
+            return true;
+
         // handle post to an individual site
-        if (request.path[1]) {
+        } else if (request.path[1]) {
             site = me.sites[request.path[1]];
 
             if (!site) {
@@ -302,7 +354,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                         console.log('Added new job');
                         console.log(site.jobs[uid]);
 
-                        // Emit maintence request with job
+                        // Emit maintenance request with job
                         me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
                     }
 
@@ -324,6 +376,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
             // apply existing site's properties in
             _.defaults(requestData, site);
+
         }
 
         // create new site

From 2c4a0f011a31faf0ba7711940f939f94a696bd64 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 18 May 2017 00:59:09 +0000
Subject: [PATCH 24/43] Add /jobs endpoint

---
 bin/kernel           |   1 +
 kernel-lib/server.js |   3 +-
 kernel-lib/sites.js  | 122 ++++++++++++++++++++++++-------------------
 3 files changed, 72 insertions(+), 54 deletions(-)

diff --git a/bin/kernel b/bin/kernel
index d231123..f92ae40 100755
--- a/bin/kernel
+++ b/bin/kernel
@@ -158,6 +158,7 @@ var eServices = require('../kernel-lib/services.js').createServices(eSites, CONF
 // instantiate management server
 var eManagementServer = require('../kernel-lib/server.js').createServer({
     sites: eSites,
+    jobs: eSites.handleJobsRequest.bind(eSites),
     services: eServices
 }, CONFIG);
 
diff --git a/kernel-lib/server.js b/kernel-lib/server.js
index 84dd2c7..a0aae1f 100644
--- a/kernel-lib/server.js
+++ b/kernel-lib/server.js
@@ -107,7 +107,8 @@ exports.Server.prototype.handleRequest = function(request, response) {
         }
 
         if (me.paths.hasOwnProperty(request.path[0])) {
-            var result = me.paths[request.path[0]].handleRequest(request, response, me);
+            var handler = me.paths[request.path[0]];
+            var result = (typeof handler == 'function' ? handler(request, response, me) : handler.handleRequest(request, response, me));
 
             if (result===false)  {
                 response.writeHead(404);
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index f2a377b..16192ae 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -194,60 +194,8 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             requestData = request.content;
         }
 
-        // handle bulk maintenance request
-        if (request.path[1] == 'maintenance') {
-
-            console.log('Received bulk maintenance request');
-            console.log(requestData);
-
-            var uid, newJobs = [];
-
-            for (a=0; a<requestData.length; a++) {
-
-                // Find site by handle
-                site = me.sites[requestData[a].handle];
-
-                if (!site) {
-                    console.error('Site not found: ' + me.sites[requestData[a].handle]);
-                    continue;
-                }
-
-                // Prune jobs
-                pruneJobs(site.jobs);
-
-                // Create uid
-                uid = uuidV1();
-
-                // Init job
-                site.jobs[uid] = {
-                    'uid': uid,
-                    'handle': requestData[a].handle,
-                    'status': 'pending',
-                    'received': new Date().getTime(),
-                    'started': null,
-                    'completed': null,
-                    'command': requestData[a],
-                };
-
-                // Append new job
-                newJobs.push(site.jobs[uid]);
-                console.log('Added new job');
-                console.log(site.jobs[uid]);
-
-                // Emit maintenance request with job
-                me.emit('maintenanceRequested', site.jobs[uid], requestData[a].handle);
-            }
-
-            response.writeHead(200, {'Content-Type':'application/json'});
-            response.end(JSON.stringify({
-                success: true,
-                message: 'maintenance request initiated',
-                jobs: newJobs
-            }));
-            return true;
-
         // handle post to an individual site
-        } else if (request.path[1]) {
+        if (request.path[1]) {
             site = me.sites[request.path[1]];
 
             if (!site) {
@@ -433,6 +381,74 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
     return false;
 };
 
+// handle bulk maintenance request
+exports.Sites.prototype.handleJobsRequest = function(request, response, server) {
+    var me = this;
+
+    console.log('Received bulk job request');
+
+    if (request.method == 'POST') {
+        var requestData, cfgResult, phpProc, phpProcInitialized, site, uid, newJobs = [];
+
+        if (request.headers['content-type'] == 'application/json') {
+            requestData = JSON.parse(request.content);
+        } else {
+            requestData = request.content;
+        }
+
+        for (a=0; a<requestData.length; a++) {
+
+            // Find site by handle
+            site = me.sites[requestData[a].handle];
+
+            if (!site) {
+                console.error('Site not found: ' + me.sites[requestData[a].handle]);
+                continue;
+            }
+
+            // Prune jobs
+            pruneJobs(site.jobs);
+
+            // Create uid
+            uid = uuidV1();
+
+            // Init job
+            site.jobs[uid] = {
+                'uid': uid,
+                'handle': requestData[a].handle,
+                'status': 'pending',
+                'received': new Date().getTime(),
+                'started': null,
+                'completed': null,
+                'command': requestData[a],
+            };
+
+            // Append new job
+            newJobs.push(site.jobs[uid]);
+            console.log('Added new job');
+            console.log(site.jobs[uid]);
+
+            // Emit maintenance request with job
+            me.emit('maintenanceRequested', site.jobs[uid], requestData[a].handle);
+        }
+
+        response.writeHead(200, {'Content-Type':'application/json'});
+        response.end(JSON.stringify({
+            success: true,
+            message: 'maintenance request initiated',
+            jobs: newJobs
+        }));
+        return true;
+    }
+
+    response.writeHead(200, {'Content-Type':'application/json'});
+    response.end(JSON.stringify({
+        success: false,
+        message: 'post job request'
+    }));
+    return true;
+}
+
 // Clean up completed jobs
 function pruneJobs(jobs) {
     var jobKeys = Object.keys(jobs),

From 99421999279660b052b5005e0f84ed1659e66602 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 18 May 2017 14:19:14 +0000
Subject: [PATCH 25/43] Use job namespace instead of maintenance

---
 kernel-lib/services/php-fpm.js             | 16 +++++------
 kernel-lib/sites.js                        | 32 +++++++++++-----------
 php-bootstrap/{maintenance.php => job.php} |  0
 3 files changed, 24 insertions(+), 24 deletions(-)
 rename php-bootstrap/{maintenance.php => job.php} (100%)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 1feef83..2da9f30 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -5,7 +5,7 @@ var _ = require('underscore'),
     spawn = require('child_process').spawn,
     phpfpm = require('node-phpfpm'),
     async = require('async'),
-    maintenenceQueue;
+    jobQueue;
 
 exports.createService = function(name, controller, options) {
     return new exports.PhpFpmService(name, controller, options);
@@ -46,8 +46,8 @@ exports.PhpFpmService = function(name, controller, options) {
         me.status = 'online';
     }
 
-    // listen for maintenance requests
-    controller.sites.on('maintenanceRequested', _.bind(me.onMaintenanceRequested, me));
+    // listen for job requests
+    controller.sites.on('jobRequested', _.bind(me.onJobRequested, me));
 };
 
 util.inherits(exports.PhpFpmService, require('./abstract.js').AbstractService);
@@ -198,7 +198,7 @@ exports.PhpFpmService.prototype.makeConfig = function() {
     return config.join('\n');
 };
 
-maintenenceQueue = async.queue(function(data, callback) {
+jobQueue = async.queue(function(data, callback) {
         var me = data.scope,
         siteRoot = me.controller.sites.options.sitesDir + '/' + data.job.handle,
         phpClient;
@@ -212,9 +212,9 @@ maintenenceQueue = async.queue(function(data, callback) {
     // Mark job as started
     data.job.started = new Date().getTime();
 
-    // Run maintenance request
+    // Run job request
     phpClient.run({
-        uri: 'maintenance.php',
+        uri: 'job.php',
         json: {
             'job': data.job,
             'handle': data.job.handle,
@@ -246,6 +246,6 @@ maintenenceQueue = async.queue(function(data, callback) {
     });
 }, 5);
 
-exports.PhpFpmService.prototype.onMaintenanceRequested = function(job) {
-    maintenenceQueue.push({'job': job, 'scope': this});
+exports.PhpFpmService.prototype.onJobRequested = function(job) {
+    jobQueue.push({'job': job, 'scope': this});
 }
diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 16192ae..41a8d6d 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -72,15 +72,15 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             }
 
             if (request.path[2]) {
-                if (request.path[2] == 'maintenance') {
-                    console.log('Received maintenance GET request for ' + request.path[1]);
+                if (request.path[2] == 'jobs') {
+                    console.log('Received jobs GET request for ' + request.path[1]);
                     response.writeHead(200, {'Content-Type':'application/json'});
 
                     // Return specific uid
                     if (request.path[3]) {
                         response.end(JSON.stringify({
                             success: true,
-                            message: 'Maintenance get request finished',
+                            message: 'Jobs get request finished',
                             jobs: (site.jobs[request.path[3]]) ? site.jobs[request.path[3]] : false
                         }));
                         return true;
@@ -89,7 +89,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                     } else {
                         response.end(JSON.stringify({
                             success: true,
-                            message: 'Maintenance get request finished',
+                            message: 'Jobs get request finished',
                             jobs: (site.jobs) ? site.jobs : false
                         }));
                         return true;
@@ -152,7 +152,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             // Create uid
             uid = uuidV1();
 
-            // Init maintenance job
+            // Init job
             site.jobs[uid] = {
                 'uid': uid,
                 'handle': request.path[1],
@@ -168,8 +168,8 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             console.log('Added new job');
             console.log(site.jobs);
 
-            // Emit maintenance request with job
-            me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
+            // Emit job request
+            me.emit('jobRequested', site.jobs[uid], request.path[1]);
 
             response.writeHead(200, {'Content-Type':'application/json'});
             response.end(JSON.stringify({
@@ -271,9 +271,9 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
 
                     return true;
 
-                } else if (request.path[2] == 'maintenance') {
+                } else if (request.path[2] == 'jobs') {
 
-                    console.log('Received maintenance request for ' + request.path[1]);
+                    console.log('Received job request for ' + request.path[1]);
                     console.log(requestData);
 
                     var uid, newJobs = [];
@@ -302,14 +302,14 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                         console.log('Added new job');
                         console.log(site.jobs[uid]);
 
-                        // Emit maintenance request with job
-                        me.emit('maintenanceRequested', site.jobs[uid], request.path[1]);
+                        // Emit job request
+                        me.emit('jobRequested', site.jobs[uid], request.path[1]);
                     }
 
                     response.writeHead(200, {'Content-Type':'application/json'});
                     response.end(JSON.stringify({
                         success: true,
-                        message: 'maintenance request initiated',
+                        message: 'job request initiated',
                         jobs: newJobs
                     }));
                     return true;
@@ -381,7 +381,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
     return false;
 };
 
-// handle bulk maintenance request
+// handle bulk job request
 exports.Sites.prototype.handleJobsRequest = function(request, response, server) {
     var me = this;
 
@@ -428,14 +428,14 @@ exports.Sites.prototype.handleJobsRequest = function(request, response, server)
             console.log('Added new job');
             console.log(site.jobs[uid]);
 
-            // Emit maintenance request with job
-            me.emit('maintenanceRequested', site.jobs[uid], requestData[a].handle);
+            // Emit job request with job
+            me.emit('jobRequested', site.jobs[uid], requestData[a].handle);
         }
 
         response.writeHead(200, {'Content-Type':'application/json'});
         response.end(JSON.stringify({
             success: true,
-            message: 'maintenance request initiated',
+            message: 'bulk job request initiated',
             jobs: newJobs
         }));
         return true;
diff --git a/php-bootstrap/maintenance.php b/php-bootstrap/job.php
similarity index 100%
rename from php-bootstrap/maintenance.php
rename to php-bootstrap/job.php

From f783bfaeece848368fc765b162810bbc76634cb3 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 18 May 2017 18:40:37 +0000
Subject: [PATCH 26/43] Handle /jobs get request

---
 kernel-lib/sites.js | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 41a8d6d..d98dbf4 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -441,10 +441,19 @@ exports.Sites.prototype.handleJobsRequest = function(request, response, server)
         return true;
     }
 
+    // Get all jobs
+    var jobs = {};
+    for (var handle in me.sites) {
+        if (Object.keys(me.sites[handle].jobs).length > 0) {
+            jobs[handle] = me.sites[handle].jobs;
+        }
+    }
+
     response.writeHead(200, {'Content-Type':'application/json'});
     response.end(JSON.stringify({
-        success: false,
-        message: 'post job request'
+        success: true,
+        message: 'get active jobs',
+        jobs: jobs
     }));
     return true;
 }

From 337c31b3ff9c0fba77ee0be7b7e61cb5ea8c1631 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 18 May 2017 22:22:29 +0000
Subject: [PATCH 27/43] Catch job.php parse errors

---
 kernel-lib/services/php-fpm.js | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 2da9f30..6abc860 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -235,12 +235,20 @@ jobQueue = async.queue(function(data, callback) {
         }
 
         // Parse job response
-        var response = JSON.parse(output);
+        try {
+            var response = JSON.parse(output);
 
-        // Update job with response
-        data.job.command = response.command;
-        data.job.status = 'completed';
-        data.job.completed = new Date().getTime();
+            // Update job with response
+            data.job.command = response.command;
+            data.job.status = 'completed';
+            data.job.completed = new Date().getTime();
+
+        } catch(e) {
+            data.job.status = 'failed';
+            data.job.message = 'PHPFPM server error';
+            console.error(output);
+            return callback(ouput);
+        }
 
         callback(null, data.job);
     });

From 07c04dd16045d4829b1bbb9ec99cef6a01226c56 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Fri, 19 May 2017 17:12:05 +0000
Subject: [PATCH 28/43] Fix site loaded console log

---
 kernel-lib/sites.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index d98dbf4..5f29e6a 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -45,7 +45,7 @@ exports.Sites = function(config) {
                 config: JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii')),
                 jobs: {}
             };
-            console.log('-Loaded: '+me.sites[handle].primary_hostname);
+            console.log('-Loaded: '+me.sites[handle].config.primary_hostname);
         } catch (error) {
             console.log('-FAILED to load: '+handle);
         }

From 4f7e490fc7ed663c8733e7104e0b3fa0bbcb4352 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Fri, 19 May 2017 17:13:02 +0000
Subject: [PATCH 29/43] Fix jobs declaration and return value

---
 kernel-lib/sites.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 5f29e6a..fc3a7c3 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -90,7 +90,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                         response.end(JSON.stringify({
                             success: true,
                             message: 'Jobs get request finished',
-                            jobs: (site.jobs) ? site.jobs : false
+                            jobs: site.jobs
                         }));
                         return true;
                     }
@@ -538,7 +538,7 @@ exports.Sites.prototype.writeSiteConfig = function(requestData) {
     this.sites[siteData.handle] = {
         handle: siteData.handle,
         config: siteData,
-        jobs: []
+        jobs: {}
     };
 
     var isNew = !fs.existsSync(siteConfigPath);

From a79472c6f9ae26316532f010a5eb58950d490efd Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 22 May 2017 16:45:21 +0000
Subject: [PATCH 30/43] Fix typo in output variable

---
 kernel-lib/services/php-fpm.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 6abc860..8442221 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -247,7 +247,7 @@ jobQueue = async.queue(function(data, callback) {
             data.job.status = 'failed';
             data.job.message = 'PHPFPM server error';
             console.error(output);
-            return callback(ouput);
+            return callback(output);
         }
 
         callback(null, data.job);

From 686bad8e47246a4e8c3b63aefb7998c5d2ef610d Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 22 May 2017 16:46:25 +0000
Subject: [PATCH 31/43] Remove 30 second job execution limit

---
 php-bootstrap/job.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/php-bootstrap/job.php b/php-bootstrap/job.php
index f77765b..a05f318 100644
--- a/php-bootstrap/job.php
+++ b/php-bootstrap/job.php
@@ -1,5 +1,7 @@
 <?php
 
+set_time_limit(0);
+
 // bootstrap emergence
 require('bootstrap.inc.php');
 
@@ -72,7 +74,7 @@ function handleVFSSummaryRequest($command, $handle, $siteRoot)
     // Initialize site for Emergence FS commands
     Site::initialize($siteRoot);
 
-    // Update the file system
+    // Get file system summary
     return getFileSystemSummary($command['cursor']);
 }
 

From 6e933498e39c0dd89a26a5db3650de38d491288b Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 22 May 2017 18:47:17 +0000
Subject: [PATCH 32/43] Fix indentation

---
 kernel-lib/services/php-fpm.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js
index 8442221..88b4ece 100644
--- a/kernel-lib/services/php-fpm.js
+++ b/kernel-lib/services/php-fpm.js
@@ -199,7 +199,7 @@ exports.PhpFpmService.prototype.makeConfig = function() {
 };
 
 jobQueue = async.queue(function(data, callback) {
-        var me = data.scope,
+    var me = data.scope,
         siteRoot = me.controller.sites.options.sitesDir + '/' + data.job.handle,
         phpClient;
 

From 2c9b583376ee80d30ea210a45ff9a10b8825864a Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 22 May 2017 18:47:53 +0000
Subject: [PATCH 33/43] Remove trailing commas

---
 kernel-lib/sites.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index fc3a7c3..266c2b2 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -294,7 +294,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
                             'received': new Date().getTime(),
                             'started': null,
                             'completed': null,
-                            'command': requestData[i],
+                            'command': requestData[i]
                         };
 
                         // Append new job
@@ -420,7 +420,7 @@ exports.Sites.prototype.handleJobsRequest = function(request, response, server)
                 'received': new Date().getTime(),
                 'started': null,
                 'completed': null,
-                'command': requestData[a],
+                'command': requestData[a]
             };
 
             // Append new job

From ebbc1694651bcfc6f1cf7ba7aef4693516fa1420 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 22 May 2017 18:48:24 +0000
Subject: [PATCH 34/43] Update command data being passed in php job processing

---
 php-bootstrap/job.php | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/php-bootstrap/job.php b/php-bootstrap/job.php
index a05f318..6a4d7a8 100644
--- a/php-bootstrap/job.php
+++ b/php-bootstrap/job.php
@@ -15,16 +15,16 @@
 switch ($data['job']['command']['action'])
 {
     case 'cache':
-        $data['job']['command']['result'] = handleCacheRequest($command, $data['handle'], $data['siteRoot']);
+        $data['job']['command']['result'] = handleCacheRequest($data['job']['command'], $data['handle'], $data['siteRoot']);
         break;
     case 'config-reload':
-        $data['job']['command']['result'] = handleConfigReload($command, $data['handle'], $data['siteRoot']);
+        $data['job']['command']['result'] = handleConfigReload($data['job']['command'], $data['handle'], $data['siteRoot']);
         break;
     case 'vfs-update':
-        $data['job']['command']['result'] = handleVFSUpdateRequest($command, $data['handle'], $data['siteRoot']);
+        $data['job']['command']['result'] = handleVFSUpdateRequest($data['job']['command'], $data['handle'], $data['siteRoot']);
         break;
     case 'vfs-summary':
-        $data['job']['command']['result'] = handleVFSSummaryRequest($command, $data['handle'], $data['siteRoot']);
+        $data['job']['command']['result'] = handleVFSSummaryRequest($data['job']['command'], $data['handle'], $data['siteRoot']);
         break;
 }
 
@@ -85,7 +85,7 @@ function handleVFSUpdateRequest($command, $handle, $siteRoot)
     Site::initialize($siteRoot);
 
     // Update the file system
-    return updateFileSystem($handle, $command['cursor'], $siteRoot);
+    return updateFileSystem($handle, $command['cursor']);
 }
 
 function updateFileSystem($handle, $cursor = 0)
@@ -128,7 +128,7 @@ function getFileSystemSummary($cursor = 0)
     $localKeys = array_keys($localFiles);
 
     // Get parent files / keys
-    $parentVFSUrl = Emergence::buildUrl();
+    $parentVFSUrl = Emergence::buildUrl([], ['minId' => $cursor]);
     $curl = curl_init();
     curl_setopt_array($curl, array(
         CURLOPT_RETURNTRANSFER => 1,

From b2a73f14060e42cf70d24af95c56799d5424df77 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Thu, 25 May 2017 16:47:52 +0000
Subject: [PATCH 35/43] Add cursor to all get file system requests

---
 php-bootstrap/job.php | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/php-bootstrap/job.php b/php-bootstrap/job.php
index 6a4d7a8..f5abfbd 100644
--- a/php-bootstrap/job.php
+++ b/php-bootstrap/job.php
@@ -75,7 +75,7 @@ function handleVFSSummaryRequest($command, $handle, $siteRoot)
     Site::initialize($siteRoot);
 
     // Get file system summary
-    return getFileSystemSummary($command['cursor']);
+    return getFileSystemSummary(intval($command['cursor']));
 }
 
 // Run full update of vfs
@@ -85,7 +85,7 @@ function handleVFSUpdateRequest($command, $handle, $siteRoot)
     Site::initialize($siteRoot);
 
     // Update the file system
-    return updateFileSystem($handle, $command['cursor']);
+    return updateFileSystem($handle, intval($command['cursor']));
 }
 
 function updateFileSystem($handle, $cursor = 0)
@@ -94,7 +94,7 @@ function updateFileSystem($handle, $cursor = 0)
     \Site::$autoPull = true;
 
     // Get update summary
-    $summary = getFileSystemSummary($cursor);
+    $summary = getFileSystemSummary(intval($cursor));
 
     // Precache files
     foreach ($summary['new'] as $path) {
@@ -128,7 +128,11 @@ function getFileSystemSummary($cursor = 0)
     $localKeys = array_keys($localFiles);
 
     // Get parent files / keys
-    $parentVFSUrl = Emergence::buildUrl([], ['minId' => $cursor]);
+    $parentVFSUrl = Emergence::buildUrl([], [
+        'minId' => $cursor,
+        'exclude' => 'sencha-workspace/(ext|touch)-.*'
+    ]);
+
     $curl = curl_init();
     curl_setopt_array($curl, array(
         CURLOPT_RETURNTRANSFER => 1,

From 568ef42a727bedc798d65e44fb079b4a781d6cd4 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Fri, 26 May 2017 14:58:22 +0000
Subject: [PATCH 36/43] Add flag for including deleted file in /emergence
 request

---
 php-bootstrap/job.php                    |  1 +
 php-bootstrap/lib/Emergence.class.php    |  8 +++++++-
 php-bootstrap/lib/Emergence_FS.class.php | 17 +++++++++++------
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/php-bootstrap/job.php b/php-bootstrap/job.php
index f5abfbd..a99cbea 100644
--- a/php-bootstrap/job.php
+++ b/php-bootstrap/job.php
@@ -130,6 +130,7 @@ function getFileSystemSummary($cursor = 0)
     // Get parent files / keys
     $parentVFSUrl = Emergence::buildUrl([], [
         'minId' => $cursor,
+        'includeDeleted' => 1,
         'exclude' => 'sencha-workspace/(ext|touch)-.*'
     ]);
 
diff --git a/php-bootstrap/lib/Emergence.class.php b/php-bootstrap/lib/Emergence.class.php
index a2c7afa..a507df2 100644
--- a/php-bootstrap/lib/Emergence.class.php
+++ b/php-bootstrap/lib/Emergence.class.php
@@ -24,6 +24,9 @@ public static function handleRequest()
             if (!empty($_REQUEST['exclude'])) {
                 $remoteParams['exclude'] = $_REQUEST['exclude'];
             }
+            if (!empty($_REQUEST['includeDeleted'])) {
+                $remoteParams['includeDeleted'] = true;
+            }
             if (!empty($_REQUEST['minId'])) {
                 $remoteParams['minId'] = $_REQUEST['minId'];
             }
@@ -82,13 +85,16 @@ public static function handleTreeRequest($rootNode = null)
             }
         }
 
+        // set include deleted
+        $includeDeleted = !empty($_REQUEST['includeDeleted']);
+
         // set minimum id
         if (!empty($_REQUEST['minId'])) {
             $fileConditions[] = 'ID > ' . intval($_REQUEST['minId']);
         }
 
         // get files
-        $files = Emergence_FS::getTreeFiles($rootPath, false, $fileConditions, $collectionConditions);
+        $files = Emergence_FS::getTreeFiles($rootPath, false, $fileConditions, $collectionConditions, $includeDeleted);
 
         header('HTTP/1.1 300 Multiple Choices');
         header('Content-Type: application/vnd.emergence.tree+json');
diff --git a/php-bootstrap/lib/Emergence_FS.class.php b/php-bootstrap/lib/Emergence_FS.class.php
index 4a6d145..47516fd 100644
--- a/php-bootstrap/lib/Emergence_FS.class.php
+++ b/php-bootstrap/lib/Emergence_FS.class.php
@@ -47,7 +47,6 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted
             $path = Site::splitPath($path);
         }
 
-
         if ($path) {
             $collections = static::getCollectionLayers($path, $localOnly);
 
@@ -84,6 +83,7 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted
             $conditions['Site'] = 'Local';
         }
 
+
         // Filter out deleted collections
         if (!$includeDeleted) {
             $conditions['Status'] = 'Normal';
@@ -128,7 +128,7 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted
             ,'SELECT ID, Site, Handle, ParentID, Status FROM `%s` WHERE (%s) ORDER BY Site = "Remote", PosLeft'
             ,array(
                 SiteCollection::$tableName
-                ,join(') AND (', $mappedConditions)
+                ,count($mappedConditions) ? join(') AND (', $mappedConditions) : '1'
             )
         );
 
@@ -139,13 +139,18 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted
         return $tree;
     }
 
-    public static function getTreeFiles($path = null, $localOnly = false, $fileConditions = array(), $collectionConditions = array()) {
-        return static::getTreeFilesFromTree(static::getTree($path, $localOnly, false, $collectionConditions), $fileConditions);
+    public static function getTreeFiles($path = null, $localOnly = false, $fileConditions = array(), $collectionConditions = array(), $includeDeleted = false) {
+        return static::getTreeFilesFromTree(static::getTree($path, $localOnly, $includeDeleted, $collectionConditions), $fileConditions, $includeDeleted);
     }
 
-    public static function getTreeFilesFromTree($tree, $conditions = array()) {
+    public static function getTreeFilesFromTree($tree, $conditions = array(), $includeDeleted = false) {
 
-        $conditions['Status'] = 'Normal';
+        // allow for includeDeleted
+        if ($includeDeleted) {
+            $conditions[] = 'Status IN ("Normal", "Deleted")';
+        } else {
+            $conditions['Status'] = 'Normal';
+        }
 
         // map conditions
         $mappedConditions = array();

From 4c560effd92f433391282cfe7923319b64bc221b Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 26 Jun 2017 12:24:11 -0400
Subject: [PATCH 37/43] Improve delete / new file reporting

---
 php-bootstrap/job.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/php-bootstrap/job.php b/php-bootstrap/job.php
index a99cbea..b5ad1f4 100644
--- a/php-bootstrap/job.php
+++ b/php-bootstrap/job.php
@@ -124,7 +124,7 @@ function updateFileSystem($handle, $cursor = 0)
 function getFileSystemSummary($cursor = 0)
 {
     // Local files / keys
-    $localFiles = Emergence_FS::getTreeFiles(null, false);
+    $localFiles = Emergence_FS::getTreeFiles(null, false, [], [], true);
     $localKeys = array_keys($localFiles);
 
     // Get parent files / keys
@@ -161,11 +161,11 @@ function getFileSystemSummary($cursor = 0)
         // and won't require the connection to the local /emergence endpoint
 
         // Find new files
-        if (!in_array($path, $localKeys)) {
+        if (!in_array($path, $localKeys) && $data['SHA1'] != null) {
             array_push($newFiles, $path);
 
         // Find deleted files
-        } elseif ($data['SHA1'] == null && $localFiles[$path]['Site'] == 'Remote') {
+        } elseif ($data['SHA1'] == null && $localFiles[$path]['Site'] == 'Remote' && $localFiles[$path]['SHA1'] != null) {
             array_push($deletedFiles, $path);
 
         // Find updated files by mismatched SHA1s

From 43b459c831b58efb39b7c616366737c83703e4c4 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 3 Jul 2017 14:26:49 -0400
Subject: [PATCH 38/43] Update logs to only show newly added jobs

---
 kernel-lib/sites.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js
index 266c2b2..4ef5e32 100644
--- a/kernel-lib/sites.js
+++ b/kernel-lib/sites.js
@@ -166,7 +166,7 @@ exports.Sites.prototype.handleRequest = function(request, response, server) {
             };
 
             console.log('Added new job');
-            console.log(site.jobs);
+            console.log(site.jobs[uid]);
 
             // Emit job request
             me.emit('jobRequested', site.jobs[uid], request.path[1]);

From 70dacb473a8e90c8cef6736338dff3f10a7bd25e Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Mon, 7 Aug 2017 10:37:44 -0400
Subject: [PATCH 39/43] Add worker_rlimit_nofile setting to nginx config

---
 kernel-lib/services/nginx.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kernel-lib/services/nginx.js b/kernel-lib/services/nginx.js
index 41a2fb2..f41d9e7 100644
--- a/kernel-lib/services/nginx.js
+++ b/kernel-lib/services/nginx.js
@@ -178,7 +178,8 @@ exports.NginxService.prototype.makeConfig = function() {
         'user '+me.options.user+' '+me.options.group+';',
         'worker_processes auto;',
         'pid '+me.options.pidPath+';',
-        'error_log '+me.options.errorLogPath+' info;'
+        'error_log '+me.options.errorLogPath+' info;',
+        'worker_rlimit_nofile 100000;'
     );
 
 

From da28cec191c60acb1a62c8934acd71e899ac13bf Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Fri, 11 Aug 2017 14:13:51 -0400
Subject: [PATCH 40/43] Set mysql config explicit_defaults_for_timestamp

Supports mysql 5.7
---
 kernel-lib/services/mysql.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kernel-lib/services/mysql.js b/kernel-lib/services/mysql.js
index ae462e2..b9aceed 100644
--- a/kernel-lib/services/mysql.js
+++ b/kernel-lib/services/mysql.js
@@ -248,6 +248,7 @@ exports.MysqlService.prototype.makeConfig = function() {
         'read_buffer_size                   = 256K',
         'read_rnd_buffer_size               = 512K',
         'myisam_sort_buffer_size            = 8M',
+        'explicit_defaults_for_timestamp    = 1',
 //      'lc-messages-dir                    = /usr/local/share/mysql',
 
         'log-bin                            = mysqld-bin',

From 66cc5f314cab74338f2535663ba7c6b054652a72 Mon Sep 17 00:00:00 2001
From: Tyler Wiest <jtwiest@gmail.com>
Date: Fri, 29 Sep 2017 09:04:57 -0400
Subject: [PATCH 41/43] Increase default mysql max_allowed_packet option

---
 kernel-lib/services/mysql.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel-lib/services/mysql.js b/kernel-lib/services/mysql.js
index b9aceed..6c5d732 100644
--- a/kernel-lib/services/mysql.js
+++ b/kernel-lib/services/mysql.js
@@ -242,7 +242,7 @@ exports.MysqlService.prototype.makeConfig = function() {
         'datadir                            = '+me.options.dataDir,
         'skip-external-locking',
         'key_buffer_size                    = 16M',
-        'max_allowed_packet                 = 1M',
+        'max_allowed_packet                 = 512M',
         'sort_buffer_size                   = 512K',
         'net_buffer_length                  = 8K',
         'read_buffer_size                   = 256K',

From ea715e147a5b800fd03cb4dfd02932945cbce1bb Mon Sep 17 00:00:00 2001
From: Chris Alfano <chris@jarv.us>
Date: Fri, 27 Apr 2018 17:46:37 +0000
Subject: [PATCH 42/43] Cherrypick bf26358530cee616ce84e2bc07c5426487f618fc

---
 bin/fire-event          | 50 +++++++++++++++++++++++++++++++++++++++++
 package.json            |  3 ++-
 php-bootstrap/event.php | 46 +++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100755 bin/fire-event
 create mode 100644 php-bootstrap/event.php

diff --git a/bin/fire-event b/bin/fire-event
new file mode 100755
index 0000000..813be47
--- /dev/null
+++ b/bin/fire-event
@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+
+
+require('yargs')
+    .command('$0 <site> <event> <context>', 'Fire event within site', yargs => {
+        yargs
+            .positional('site', {
+                describe: 'Handle of site to fire event within'
+            })
+            .positional('event', {
+                describe: 'Name of event to fire'
+            })
+            .positional('context', {
+                describe: 'Context path to fire event within'
+            })
+    }, argv => {
+        var path = require('path'),
+            documentRoot = path.resolve(__dirname, '../php-bootstrap'),
+            PHPFPM = require('node-phpfpm'),
+            phpClient = new PHPFPM({
+                sockFile: '/emergence/services/run/php-fpm/php-fpm.sock',
+                documentRoot: documentRoot + '/'
+            }),
+            payload = Object.assign({}, argv);
+
+        delete payload._;
+        delete payload.help;
+        delete payload.version;
+        delete payload.site;
+        delete payload.event;
+        delete payload.context;
+        delete payload['$0'];
+
+        // execute event via PHP-FPM interface
+        phpClient.run({
+            uri: 'event.php',
+            json: {
+                site: argv.site,
+                event: argv.event,
+                context: argv.context,
+                payload: payload
+            }
+        }, function (err, output, phpErrors) {
+            if (err == 99) console.error('PHPFPM server error');
+            console.log(output);
+            if (phpErrors) console.error(phpErrors);
+        });
+
+    })
+    .argv;
diff --git a/package.json b/package.json
index 82edbc1..c0bc6ff 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,8 @@
     "underscore-cli": "^0.2.19",
     "node-phpfpm": "^1.0.1",
     "uuid": "~3.0.1",
-    "async": "~2.4.0"
+    "async": "~2.4.0",
+    "yargs": "^11.0.0"
   },
   "bin": {
     "emergence-kernel": "./bin/kernel",
diff --git a/php-bootstrap/event.php b/php-bootstrap/event.php
new file mode 100644
index 0000000..9eca178
--- /dev/null
+++ b/php-bootstrap/event.php
@@ -0,0 +1,46 @@
+<?php
+
+// bootstrap emergence
+require('bootstrap.inc.php');
+
+
+// only process POST requests
+if ($_SERVER['REQUEST_METHOD'] != 'POST') {
+    http_response_code(405);
+    echo "POST only";
+    exit(1);
+}
+
+
+// parse and validate input
+$input = json_decode(file_get_contents('php://input'), true);
+
+if (
+    empty($input)
+    || empty($input['site'])
+    || empty($input['event'])
+    || empty($input['context'])
+) {
+    http_response_code(400);
+    echo "site, event, and context required";
+    exit(1);
+}
+
+
+// identify site
+$siteRoot = "/emergence/sites/{$input['site']}";
+
+if (!is_dir($siteRoot)) {
+    http_response_code(404);
+    echo "site not found";
+    exit(1);
+}
+
+
+// initialize site
+Site::$debug = true;
+Site::initialize($siteRoot, 'localhost');
+
+
+// fire event
+Emergence\EventBus::fireEvent($input['event'], $input['context'], !empty($input['payload']) ? $input['payload'] : array());

From 09c0b60cf88f930a426c98c1dd82e156c78a28cb Mon Sep 17 00:00:00 2001
From: Chris Alfano <chris@jarv.us>
Date: Fri, 27 Apr 2018 19:27:22 +0000
Subject: [PATCH 43/43] Register emergence-fire-event command

---
 package.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index c0bc6ff..300e6f4 100644
--- a/package.json
+++ b/package.json
@@ -26,7 +26,8 @@
     "emergence-git-shell": "./bin/git-shell",
     "emergence-su": "./bin/su",
     "emergence-create-site": "./bin/create-site.sh",
-    "emergence-mysql-shell": "./bin/mysql-shell"
+    "emergence-mysql-shell": "./bin/mysql-shell",
+    "emergence-fire-event": "./bin/fire-event"
   },
   "repository": {
     "type": "git",