diff --git a/bin/fire-event b/bin/fire-event new file mode 100755 index 0000000..813be47 --- /dev/null +++ b/bin/fire-event @@ -0,0 +1,50 @@ +#!/usr/bin/env node + + +require('yargs') + .command('$0 ', 'Fire event within site', yargs => { + yargs + .positional('site', { + describe: 'Handle of site to fire event within' + }) + .positional('event', { + describe: 'Name of event to fire' + }) + .positional('context', { + describe: 'Context path to fire event within' + }) + }, argv => { + var path = require('path'), + documentRoot = path.resolve(__dirname, '../php-bootstrap'), + PHPFPM = require('node-phpfpm'), + phpClient = new PHPFPM({ + sockFile: '/emergence/services/run/php-fpm/php-fpm.sock', + documentRoot: documentRoot + '/' + }), + payload = Object.assign({}, argv); + + delete payload._; + delete payload.help; + delete payload.version; + delete payload.site; + delete payload.event; + delete payload.context; + delete payload['$0']; + + // execute event via PHP-FPM interface + phpClient.run({ + uri: 'event.php', + json: { + site: argv.site, + event: argv.event, + context: argv.context, + payload: payload + } + }, function (err, output, phpErrors) { + if (err == 99) console.error('PHPFPM server error'); + console.log(output); + if (phpErrors) console.error(phpErrors); + }); + + }) + .argv; diff --git a/bin/kernel b/bin/kernel index 97bd81d..006fd4a 100755 --- a/bin/kernel +++ b/bin/kernel @@ -158,6 +158,7 @@ var eServices = require('../kernel-lib/services.js').createServices(eSites, CONF // instantiate management server var eManagementServer = require('../kernel-lib/server.js').createServer({ sites: eSites, + jobs: eSites.handleJobsRequest.bind(eSites), services: eServices }, CONFIG); diff --git a/kernel-lib/server.js b/kernel-lib/server.js index 820f386..565d3a7 100644 --- a/kernel-lib/server.js +++ b/kernel-lib/server.js @@ -107,7 +107,8 @@ exports.Server.prototype.handleRequest = function(request, response) { } if (me.paths.hasOwnProperty(request.path[0])) { - var result = me.paths[request.path[0]].handleRequest(request, response, me); + var handler = me.paths[request.path[0]]; + var result = (typeof handler == 'function' ? handler(request, response, me) : handler.handleRequest(request, response, me)); if (result===false) { response.writeHead(404); diff --git a/kernel-lib/services/mysql.js b/kernel-lib/services/mysql.js index 7e9bba2..fc0af41 100644 --- a/kernel-lib/services/mysql.js +++ b/kernel-lib/services/mysql.js @@ -220,12 +220,13 @@ exports.MysqlService.prototype.makeConfig = function() { 'datadir = '+me.options.dataDir, 'skip-external-locking', 'key_buffer_size = 16M', - 'max_allowed_packet = 1M', + 'max_allowed_packet = 512M', 'sort_buffer_size = 512K', 'net_buffer_length = 8K', 'read_buffer_size = 256K', 'read_rnd_buffer_size = 512K', 'myisam_sort_buffer_size = 8M', + 'explicit_defaults_for_timestamp = 1', // 'lc-messages-dir = /usr/local/share/mysql', 'log-bin = mysqld-bin', diff --git a/kernel-lib/services/nginx.js b/kernel-lib/services/nginx.js index b42aba3..f41d9e7 100644 --- a/kernel-lib/services/nginx.js +++ b/kernel-lib/services/nginx.js @@ -178,7 +178,8 @@ exports.NginxService.prototype.makeConfig = function() { 'user '+me.options.user+' '+me.options.group+';', 'worker_processes auto;', 'pid '+me.options.pidPath+';', - 'error_log '+me.options.errorLogPath+' info;' + 'error_log '+me.options.errorLogPath+' info;', + 'worker_rlimit_nofile 100000;' ); @@ -256,15 +257,15 @@ exports.NginxService.prototype.makeConfig = function() { ); _.each(me.controller.sites.sites, function(site, handle) { - var hostnames = site.hostnames.slice(), + var hostnames = site.config.hostnames.slice(), siteDir = me.controller.sites.options.sitesDir+'/'+handle, logsDir = siteDir+'/logs', siteConfig = [], sslHostnames, sslHostname; // process hostnames - if (_.indexOf(hostnames, site.primary_hostname) == -1) { - hostnames.unshift(site.primary_hostname); + if (_.indexOf(hostnames, site.config.primary_hostname) == -1) { + hostnames.unshift(site.config.primary_hostname); } // process directories @@ -301,15 +302,15 @@ exports.NginxService.prototype.makeConfig = function() { ' }' ); - if (site.ssl) { - if (site.ssl.hostnames) { - sslHostnames = site.ssl.hostnames; + if (site.config.ssl) { + if (site.config.ssl.hostnames) { + sslHostnames = site.config.ssl.hostnames; } else { sslHostnames = {}; - sslHostnames[site.primary_hostname] = site.ssl; + sslHostnames[site.config.primary_hostname] = site.config.ssl; - site.hostnames.forEach(function(hostname) { - sslHostnames[hostname] = site.ssl; + site.config.hostnames.forEach(function(hostname) { + sslHostnames[hostname] = site.config.ssl; }); } diff --git a/kernel-lib/services/php-fpm.js b/kernel-lib/services/php-fpm.js index 9cde0fd..3b7942b 100644 --- a/kernel-lib/services/php-fpm.js +++ b/kernel-lib/services/php-fpm.js @@ -3,7 +3,9 @@ var _ = require('underscore'), path = require('path'), util = require('util'), spawn = require('child_process').spawn, - phpfpm = require('node-phpfpm'); + phpfpm = require('node-phpfpm'), + async = require('async'), + jobQueue; exports.createService = function(name, controller, options) { return new exports.PhpFpmService(name, controller, options); @@ -44,8 +46,8 @@ exports.PhpFpmService = function(name, controller, options) { me.status = 'online'; } - // listen for site updated - controller.sites.on('siteUpdated', _.bind(me.onSiteUpdated, me)); + // listen for job requests + controller.sites.on('jobRequested', _.bind(me.onJobRequested, me)); }; util.inherits(exports.PhpFpmService, require('./abstract.js').AbstractService); @@ -197,28 +199,62 @@ exports.PhpFpmService.prototype.makeConfig = function() { return config.join('\n'); }; -exports.PhpFpmService.prototype.onSiteUpdated = function(siteData) { - var me = this, - siteRoot = me.controller.sites.options.sitesDir + '/' + siteData.handle, +jobQueue = async.queue(function(data, callback) { + var me = data.scope, + siteRoot = me.controller.sites.options.sitesDir + '/' + data.job.handle, phpClient; - console.log(me.name+': clearing config cache for '+siteRoot); - // Connect to FPM worker pool phpClient = new phpfpm({ sockFile: me.options.socketPath, documentRoot: me.options.bootstrapDir + '/' }); - // Clear cached site.json + // Mark job as started + data.job.started = new Date().getTime(); + + // Run job request phpClient.run({ - uri: 'cache.php', - json: [ - { action: 'delete', key: siteRoot } - ] - }, function(err, output, phpErrors) { - if (err == 99) console.error('PHPFPM server error'); - console.log(output); - if (phpErrors) console.error(phpErrors); + uri: 'job.php', + json: { + 'job': data.job, + 'handle': data.job.handle, + 'siteRoot': siteRoot + } + }, function(err, output, stderr) { + if (err == 99) { + data.job.status = 'failed'; + data.job.message = 'PHPFPM server error'; + console.error(data.job.message); + return callback(err); + } + if (stderr) { + data.job.status = 'failed'; + data.job.message = stderr; + console.error(stderr); + return callback(stderr); + } + + // Parse job response + try { + var response = JSON.parse(output); + + // Update job with response + data.job.command = response.command; + data.job.status = 'completed'; + data.job.completed = new Date().getTime(); + + } catch(e) { + data.job.status = 'failed'; + data.job.message = 'PHPFPM server error'; + console.error(output); + return callback(output); + } + + callback(null, data.job); }); -}; +}, 5); + +exports.PhpFpmService.prototype.onJobRequested = function(job) { + jobQueue.push({'job': job, 'scope': this}); +} diff --git a/kernel-lib/sites.js b/kernel-lib/sites.js index 60e2458..c8e9211 100644 --- a/kernel-lib/sites.js +++ b/kernel-lib/sites.js @@ -7,7 +7,8 @@ var _ = require('underscore'), posix = require('posix'), spawn = require('child_process').spawn, hostile = require('hostile'), - phpShellScript = path.resolve(__dirname, '../bin/shell'); + phpShellScript = path.resolve(__dirname, '../bin/shell'), + uuidV1 = require('uuid/v1'); exports.createSites = function(config) { @@ -40,9 +41,12 @@ exports.Sites = function(config) { me.sites = {}; _.each(fs.readdirSync(me.options.sitesDir), function(handle) { try { - me.sites[handle] = JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii')); - me.sites[handle].handle = handle; - console.log('-Loaded: '+me.sites[handle].primary_hostname); + me.sites[handle] = { + handle: handle, + config: JSON.parse(fs.readFileSync(me.options.sitesDir+'/'+handle+'/site.json', 'ascii')), + jobs: {} + }; + console.log('-Loaded: '+me.sites[handle].config.primary_hostname); } catch (error) { console.log('-FAILED to load: '+handle); } @@ -52,34 +56,69 @@ exports.Sites = function(config) { util.inherits(exports.Sites, events.EventEmitter); - exports.Sites.prototype.handleRequest = function(request, response, server) { - var me = this; + var me = this, + site; if (request.method == 'GET') { if (request.path[1]) { - if (!me.sites[request.path[1]]) { + site = me.sites[request.path[1]]; + + if (!site) { console.error('Site not found: ' + request.path[1]); response.writeHead(404, {'Content-Type':'application/json'}); response.end(JSON.stringify({success: false, message: 'Site not found'})); return; } + if (request.path[2]) { + if (request.path[2] == 'jobs') { + console.log('Received jobs GET request for ' + request.path[1]); + response.writeHead(200, {'Content-Type':'application/json'}); + + // Return specific uid + if (request.path[3]) { + response.end(JSON.stringify({ + success: true, + message: 'Jobs get request finished', + jobs: (site.jobs[request.path[3]]) ? site.jobs[request.path[3]] : false + })); + return true; + + // Return all jobs + } else { + response.end(JSON.stringify({ + success: true, + message: 'Jobs get request finished', + jobs: site.jobs + })); + return true; + } + + } else { + console.error('Unhandled site sub-resource: ' + request.path[2]); + response.writeHead(404, {'Content-Type':'application/json'}); + response.end(JSON.stringify({success: false, message: 'Site resource not found'})); + return; + } + } + response.writeHead(200, {'Content-Type':'application/json'}); - response.end(JSON.stringify({data: me.sites[request.path[1]]})); + response.end(JSON.stringify({data: site.config})); return true; } response.writeHead(200, {'Content-Type':'application/json'}); - response.end(JSON.stringify({data: _.values(me.sites)})); + response.end(JSON.stringify({data: _.pluck(_.values(me.sites), 'config')})); return true; } else if (request.method == 'PATCH') { if (request.path[1]) { + site = me.sites[request.path[1]]; - if (!me.sites[request.path[1]]) { + if (!site) { console.error('Site not found: ' + request.path[1]); response.writeHead(404, {'Content-Type':'application/json'}); response.end(JSON.stringify({success: false, message: 'Site not found'})); @@ -90,28 +129,54 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { siteDir = me.options.sitesDir + '/' + handle, siteConfigPath = siteDir + '/site.json', params = JSON.parse(request.content), - siteData; + siteData, siteDataTmp, uid; // Get existing site config siteData = me.sites[handle]; + // Prune jobs + pruneJobs(siteData.jobs); + // Apply updates except handle for (var k in params) { if (k !== 'handle') { - siteData[k] = params[k]; + siteData.config[k] = params[k]; } } - // Update file - fs.writeFileSync(siteConfigPath, JSON.stringify(siteData, null, 4)); + // Update config file + fs.writeFileSync(siteConfigPath, JSON.stringify(siteData.config, null, 4)); // Restart nginx me.emit('siteUpdated', siteData); - response.writeHead(404, {'Content-Type':'application/json'}); + // Create uid + uid = uuidV1(); + + // Init job + site.jobs[uid] = { + 'uid': uid, + 'handle': request.path[1], + 'status': 'pending', + 'received': new Date().getTime(), + 'started': null, + 'completed': null, + 'command': { + 'action': 'config-reload' + } + }; + + console.log('Added new job'); + console.log(site.jobs[uid]); + + // Emit job request + me.emit('jobRequested', site.jobs[uid], request.path[1]); + + response.writeHead(200, {'Content-Type':'application/json'}); response.end(JSON.stringify({ success: true, message: 'Processed patch request', + job: site.jobs[uid] })); return; } @@ -132,7 +197,9 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { // handle post to an individual site if (request.path[1]) { - if (!me.sites[request.path[1]]) { + site = me.sites[request.path[1]]; + + if (!site) { console.error('Site not found: ' + request.path[1]); response.writeHead(404, {'Content-Type':'application/json'}); response.end(JSON.stringify({success: false, message: 'Site not found'})); @@ -204,6 +271,50 @@ exports.Sites.prototype.handleRequest = function(request, response, server) { }); return true; + + } else if (request.path[2] == 'jobs') { + + console.log('Received job request for ' + request.path[1]); + console.log(requestData); + + var uid, newJobs = []; + + // Prune jobs + pruneJobs(site.jobs); + + for (i=0; i 0) { + jobs[handle] = me.sites[handle].jobs; + } + } + + response.writeHead(200, {'Content-Type':'application/json'}); + response.end(JSON.stringify({ + success: true, + message: 'get active jobs', + jobs: jobs + })); + return true; +} + +// Clean up completed jobs +function pruneJobs(jobs) { + var jobKeys = Object.keys(jobs), + cutoffTime = new Date().getTime() - (60 * 60 * 1000); // 1 hour ago + + for (i=0; idelete(); + } + } + + // Update files + foreach ($summary['updated'] as $path) { + $Node = \Site::resolvePath($path); + $NewNode = \Emergence::resolveFileFromParent($Node->Collection, $Node->Handle, true); + } + + // Get updated local cursor + $summary['localCursor'] = getLocalCursor(); + + return $summary; +} + +// Retrieve the file summary for given site +function getFileSystemSummary($cursor = 0) +{ + // Local files / keys + $localFiles = Emergence_FS::getTreeFiles(null, false, [], [], true); + $localKeys = array_keys($localFiles); + + // Get parent files / keys + $parentVFSUrl = Emergence::buildUrl([], [ + 'minId' => $cursor, + 'includeDeleted' => 1, + 'exclude' => 'sencha-workspace/(ext|touch)-.*' + ]); + + $curl = curl_init(); + curl_setopt_array($curl, array( + CURLOPT_RETURNTRANSFER => 1, + CURLOPT_URL => $parentVFSUrl + )); + $parentFileResponse = json_decode(curl_exec($curl), true); + $parentFiles = $parentFileResponse['files']; + $parentKeys = array_keys($parentFiles); + + $newFiles = []; + $updatedFiles = []; + $deletedFiles = []; + $parentCursor = $cursor; + + // Compare remote files against their local copies + foreach ($parentFiles as $path => $data) { + + // Update parent cursor + if ($data['ID'] > $parentCursor) { + $parentCursor = $data['ID']; + } + + // @todo instead of comparing against local keys, do real time queries + // for local nodes. This will minimize the number of total queries run + // and won't require the connection to the local /emergence endpoint + + // Find new files + if (!in_array($path, $localKeys) && $data['SHA1'] != null) { + array_push($newFiles, $path); + + // Find deleted files + } elseif ($data['SHA1'] == null && $localFiles[$path]['Site'] == 'Remote' && $localFiles[$path]['SHA1'] != null) { + array_push($deletedFiles, $path); + + // Find updated files by mismatched SHA1s + } elseif ($data['SHA1'] !== $localFiles[$path]['SHA1'] && $localFiles[$path]['Site'] == 'Remote') { + array_push($updatedFiles, $path); + } + } + + return [ + 'new' => $newFiles, + 'updated' => $updatedFiles, + 'deleted' => $deletedFiles, + 'parentCursor' => $parentCursor, + 'localCursor' => getLocalCursor() + ]; +} + +// Retrieve the local cursor for the given site +function getLocalCursor() +{ + return \DB::oneValue('SELECT MAX(ID) FROM _e_files'); +} diff --git a/php-bootstrap/lib/Emergence.class.php b/php-bootstrap/lib/Emergence.class.php index 55ac296..f975ef7 100644 --- a/php-bootstrap/lib/Emergence.class.php +++ b/php-bootstrap/lib/Emergence.class.php @@ -24,6 +24,9 @@ public static function handleRequest() if (!empty($_REQUEST['exclude'])) { $remoteParams['exclude'] = $_REQUEST['exclude']; } + if (!empty($_REQUEST['includeDeleted'])) { + $remoteParams['includeDeleted'] = true; + } if (!empty($_REQUEST['minId'])) { $remoteParams['minId'] = $_REQUEST['minId']; } @@ -82,13 +85,16 @@ public static function handleTreeRequest($rootNode = null) } } + // set include deleted + $includeDeleted = !empty($_REQUEST['includeDeleted']); + // set minimum id if (!empty($_REQUEST['minId'])) { $fileConditions[] = 'ID > ' . intval($_REQUEST['minId']); } // get files - $files = Emergence_FS::getTreeFiles($rootPath, false, $fileConditions, $collectionConditions); + $files = Emergence_FS::getTreeFiles($rootPath, false, $fileConditions, $collectionConditions, $includeDeleted); header('HTTP/1.1 300 Multiple Choices'); header('Content-Type: application/vnd.emergence.tree+json'); diff --git a/php-bootstrap/lib/Emergence_FS.class.php b/php-bootstrap/lib/Emergence_FS.class.php index a0cb2b8..e10bef5 100644 --- a/php-bootstrap/lib/Emergence_FS.class.php +++ b/php-bootstrap/lib/Emergence_FS.class.php @@ -47,7 +47,6 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted $path = Site::splitPath($path); } - if ($path) { $collections = static::getCollectionLayers($path, $localOnly); @@ -84,6 +83,7 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted $conditions['Site'] = 'Local'; } + // Filter out deleted collections if (!$includeDeleted) { $conditions['Status'] = 'Normal'; @@ -128,7 +128,7 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted ,'SELECT ID, Site, Handle, ParentID, Status FROM `%s` WHERE (%s) ORDER BY Site = "Remote", PosLeft' ,array( SiteCollection::$tableName - ,join(') AND (', $mappedConditions) + ,count($mappedConditions) ? join(') AND (', $mappedConditions) : '1' ) ); @@ -139,13 +139,18 @@ public static function getTree($path = null, $localOnly = false, $includeDeleted return $tree; } - public static function getTreeFiles($path = null, $localOnly = false, $fileConditions = array(), $collectionConditions = array()) { - return static::getTreeFilesFromTree(static::getTree($path, $localOnly, false, $collectionConditions), $fileConditions); + public static function getTreeFiles($path = null, $localOnly = false, $fileConditions = array(), $collectionConditions = array(), $includeDeleted = false) { + return static::getTreeFilesFromTree(static::getTree($path, $localOnly, $includeDeleted, $collectionConditions), $fileConditions, $includeDeleted); } - public static function getTreeFilesFromTree($tree, $conditions = array()) { + public static function getTreeFilesFromTree($tree, $conditions = array(), $includeDeleted = false) { - $conditions['Status'] = 'Normal'; + // allow for includeDeleted + if ($includeDeleted) { + $conditions[] = 'Status IN ("Normal", "Deleted")'; + } else { + $conditions['Status'] = 'Normal'; + } // map conditions $mappedConditions = array();