From 41ad395eee2f63fa9829cad5391f7ed1ba5c858d Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 31 Dec 2020 22:08:19 -0600
Subject: [PATCH 01/36] Improve init logs

---
 src/plugins/default/init.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/plugins/default/init.js b/src/plugins/default/init.js
index fa6c8ec4..aa531ff9 100644
--- a/src/plugins/default/init.js
+++ b/src/plugins/default/init.js
@@ -99,8 +99,10 @@ export default function init(api) {
     console.log('  Available options can be found in the docs at');
     console.log('    https://github.com/zodern/meteor-up');
     console.log('');
-    console.log('  Then, run the command:');
+    console.log('  Once the config is ready, you can setup your servers with:');
     console.log('    mup setup');
+    console.log('  And deploy your app by running:');
+    console.log('    mup deploy');
   } else {
     console.log('Skipping creation of mup.js');
     console.log(`mup.js already exists at ${configDest}`);

From 8fd6e5bacf26de16577b551639a17035ee3a65ca Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 31 Dec 2020 22:23:53 -0600
Subject: [PATCH 02/36] Change default image to zodern:0.6.1-root This image
 should be compatible with the old default, kadirahq/meteord (which was
 compatible up to Meteor 1.3), while simultaneously supporting all new Meteor
 versions.

---
 src/plugins/default/template/mup.js.sample | 7 -------
 src/plugins/meteor/index.js                | 2 +-
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/plugins/default/template/mup.js.sample b/src/plugins/default/template/mup.js.sample
index 18b978a1..504698f3 100644
--- a/src/plugins/default/template/mup.js.sample
+++ b/src/plugins/default/template/mup.js.sample
@@ -31,13 +31,6 @@ module.exports = {
       MONGO_OPLOG_URL: 'mongodb://mongodb/local',
     },
 
-    docker: {
-      // abernix/meteord:node-12-base works with Meteor 1.9 - 1.10
-      // If you are using a different version of Meteor,
-      // refer to the docs for the correct image to use.
-      image: 'abernix/meteord:node-12-base',
-    },
-
     // Show progress bar while uploading bundle to server
     // You might need to disable it on CI servers
     enableUploadProgressBar: true
diff --git a/src/plugins/meteor/index.js b/src/plugins/meteor/index.js
index 9a170863..d859930c 100644
--- a/src/plugins/meteor/index.js
+++ b/src/plugins/meteor/index.js
@@ -26,7 +26,7 @@ export function prepareConfig(config) {
   }
 
   config.app.docker = defaultsDeep(config.app.docker, {
-    image: config.app.dockerImage || 'kadirahq/meteord',
+    image: config.app.dockerImage || 'zodern/meteor:0.6.1-root',
     stopAppDuringPrepareBundle: true
   });
 

From 7ecef9422bdfa4e6904f7ba4d60702884afdc13a Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 31 Dec 2020 22:34:34 -0600
Subject: [PATCH 03/36] Always set upload progress bar to true

---
 CHANGELOG.md                               | 2 +-
 docs/docs.md                               | 3 +--
 src/plugins/default/template/mup.js.sample | 5 +----
 src/plugins/meteor/command-handlers.js     | 2 +-
 src/plugins/meteor/validate.js             | 8 ++++++++
 5 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 40141009..fb8d892c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -461,7 +461,7 @@ Big thanks to @shaiamir for his work on the shared proxy.
 - Change mongo version
 - Validates `mup.js` and displays problems found in it
 - Update message is clearer and more colorful
-- `uploadProgressBar` is part of default `mup.js`
+- `enableUploadProgressBar` is part of default `mup.js`
 - Add trailing commas to mup.js (@ffxsam)
 - Improve message when settings.json is not found or is invalid
 - Loads and parses settings.json before building the app
diff --git a/docs/docs.md b/docs/docs.md
index 78a3d886..9e523181 100644
--- a/docs/docs.md
+++ b/docs/docs.md
@@ -239,8 +239,7 @@ module.exports = {
     deployCheckPort: 80,
 
     // Shows progress bar while uploading bundle to server
-    // You might need to disable it on CI servers
-    // (optional, default is false)
+    // Since Meteor Up 1.6, this option does nothing, and is always true.
     enableUploadProgressBar: true
   },
 
diff --git a/src/plugins/default/template/mup.js.sample b/src/plugins/default/template/mup.js.sample
index 504698f3..4641ae9c 100644
--- a/src/plugins/default/template/mup.js.sample
+++ b/src/plugins/default/template/mup.js.sample
@@ -20,6 +20,7 @@ module.exports = {
     },
 
     buildOptions: {
+      // Set to true to skip building mobile apps
       serverOnly: true,
     },
 
@@ -30,10 +31,6 @@ module.exports = {
       MONGO_URL: 'mongodb://mongodb/meteor',
       MONGO_OPLOG_URL: 'mongodb://mongodb/local',
     },
-
-    // Show progress bar while uploading bundle to server
-    // You might need to disable it on CI servers
-    enableUploadProgressBar: true
   },
 
   mongo: {
diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js
index bfa24585..782266f0 100644
--- a/src/plugins/meteor/command-handlers.js
+++ b/src/plugins/meteor/command-handlers.js
@@ -151,7 +151,7 @@ export async function push(api) {
   list.copy('Pushing Meteor App Bundle to the Server', {
     src: bundlePath,
     dest: `/opt/${appConfig.name}/tmp/bundle.tar.gz`,
-    progressBar: appConfig.enableUploadProgressBar
+    progressBar: true
   });
 
   if (prepareBundleSupported(appConfig.docker)) {
diff --git a/src/plugins/meteor/validate.js b/src/plugins/meteor/validate.js
index da9c70ed..dd0fa2db 100644
--- a/src/plugins/meteor/validate.js
+++ b/src/plugins/meteor/validate.js
@@ -150,5 +150,13 @@ export default function(
     );
   }
 
+  if (config.app.enableUploadProgressBar) {
+    details = addDepreciation(
+      details,
+      'enableUploadProgressBar',
+      'This option is no longer used.'
+    );
+  }
+
   return addLocation(details, config.meteor ? 'meteor' : 'app');
 }

From 8d608d774f6fe8a13f6ac1c08517611f933d26e1 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 31 Dec 2020 22:41:05 -0600
Subject: [PATCH 04/36] Fix depreciation message when there is no link

---
 src/validate/utils.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/validate/utils.js b/src/validate/utils.js
index 7a056b97..5250b459 100644
--- a/src/validate/utils.js
+++ b/src/validate/utils.js
@@ -66,10 +66,12 @@ export function serversExist(serversConfig = {}, serversUsed = {}) {
 }
 
 export function addDepreciation(details, path, reason, link) {
+  const learnMore = link ? `\n Learn more at ${link}` : '';
+
   details.push({
     type: 'depreciation',
     path,
-    message: `${reason}\n  Learn more at ${link}`
+    message: `${reason}${learnMore}`
   });
 
   return details;

From f07c9ba7b60b9a0480f6fdccd06a7af7dfc11aef Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 31 Dec 2020 22:55:19 -0600
Subject: [PATCH 05/36] Expand explanations in sample config

---
 src/plugins/default/template/mup.js.sample | 39 ++++++++++++++++------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/src/plugins/default/template/mup.js.sample b/src/plugins/default/template/mup.js.sample
index 4641ae9c..73d6a68b 100644
--- a/src/plugins/default/template/mup.js.sample
+++ b/src/plugins/default/template/mup.js.sample
@@ -1,17 +1,25 @@
 module.exports = {
   servers: {
     one: {
-      // TODO: set host address, username, and authentication method
+      // TODO: set host address and username
       host: '1.2.3.4',
       username: 'root',
-      // pem: './path/to/pem'
+
+      // TODO: Choose one of these authentication methods
+      // - To use a private SSH key, set the `pem` option to the
+      //   path to the key.
+      // - For a password, set the `password` option
+      //   to the user's password.
+      // - If neither `pem` or `password` are set, the ssh-agent is used
+      //   for authentication.
+      //
+      // pem: '~/.ssh/key-name'
       // password: 'server-password'
-      // or neither for authenticate from ssh-agent
     }
   },
 
   app: {
-    // TODO: change app name and path
+    // TODO: change app name and the path to the app
     name: 'app',
     path: '<app path>',
 
@@ -25,8 +33,10 @@ module.exports = {
     },
 
     env: {
-      // TODO: Change to your app's url
-      // If you are using ssl, it needs to start with https://
+      // TODO: Change to your app's url. This should be
+      // the url you access your app at, including the correct
+      // protocol (https:// or http://) and, if you've changed it
+      // from the default, the port.
       ROOT_URL: 'http://app.com',
       MONGO_URL: 'mongodb://mongodb/meteor',
       MONGO_OPLOG_URL: 'mongodb://mongodb/local',
@@ -41,15 +51,22 @@ module.exports = {
   },
 
   // (Optional)
-  // Use the proxy to setup ssl or to route requests to the correct
-  // app when there are several apps
-
+  // Use the proxy to:
+  // - setup ssl
+  // - if multiple apps share the server, route requests to the correct one
+  // - load balance requests across multiple servers
+  //
   // proxy: {
   //   domains: 'mywebsite.com,www.mywebsite.com',
 
   //   ssl: {
   //     // Enable Let's Encrypt
-  //     letsEncryptEmail: 'email@domain.com'
-  //   }
+  //     // This email is used by Let's encrypt to
+  //     // inform you when a certificate is close to
+  //     // expiring.
+  //     letsEncryptEmail: 'email@domain.com',
+  //   },
+  //
+  //   loadBalancing: true
   // }
 };

From 4347630c6a8d29f0dc266f3e2105ba45c3b7df49 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Sun, 19 Sep 2021 21:24:11 -0500
Subject: [PATCH 06/36] Minor improvements to default config

---
 src/plugins/default/template/mup.js.sample | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/plugins/default/template/mup.js.sample b/src/plugins/default/template/mup.js.sample
index 73d6a68b..751c2ea9 100644
--- a/src/plugins/default/template/mup.js.sample
+++ b/src/plugins/default/template/mup.js.sample
@@ -6,10 +6,8 @@ module.exports = {
       username: 'root',
 
       // TODO: Choose one of these authentication methods
-      // - To use a private SSH key, set the `pem` option to the
-      //   path to the key.
-      // - For a password, set the `password` option
-      //   to the user's password.
+      // - To use a private SSH key, set the `pem` option to the path to the key
+      // - For a password, set the `password` option to the user's password
       // - If neither `pem` or `password` are set, the ssh-agent is used
       //   for authentication.
       //
@@ -38,6 +36,7 @@ module.exports = {
       // protocol (https:// or http://) and, if you've changed it
       // from the default, the port.
       ROOT_URL: 'http://app.com',
+
       MONGO_URL: 'mongodb://mongodb/meteor',
       MONGO_OPLOG_URL: 'mongodb://mongodb/local',
     },

From 962c61a756ff62bad08094d8737d78d0b6b53a3f Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 22 Sep 2021 16:41:30 -0500
Subject: [PATCH 07/36] Basic implementation of server groups

---
 src/index.js                                  |   4 +-
 src/load-plugins.js                           |   6 +
 src/plugin-api.js                             | 120 +++++++++++++---
 src/plugins/default/command-handlers.js       |   6 +-
 src/plugins/default/index.js                  |   3 +
 .../default/server-groups/digital-ocean.js    | 134 ++++++++++++++++++
 src/plugins/default/server-groups/index.js    |  47 ++++++
 src/plugins/default/server-groups/utils.js    |   7 +
 src/server-sources.js                         |   9 ++
 9 files changed, 312 insertions(+), 24 deletions(-)
 create mode 100644 src/plugins/default/server-groups/digital-ocean.js
 create mode 100644 src/plugins/default/server-groups/index.js
 create mode 100644 src/plugins/default/server-groups/utils.js
 create mode 100644 src/server-sources.js

diff --git a/src/index.js b/src/index.js
index bd8f3d2b..f119b174 100644
--- a/src/index.js
+++ b/src/index.js
@@ -40,7 +40,7 @@ if (config.hooks) {
 }
 
 function commandWrapper(pluginName, commandName) {
-  return function() {
+  return async function() {
     // Runs in parallel with command
     checkUpdates([
       { name: pkg.name, path: require.resolve('../package.json') },
@@ -52,6 +52,8 @@ function commandWrapper(pluginName, commandName) {
     const api = new MupAPI(process.cwd(), filteredArgv, yargs.argv);
     let potentialPromise;
 
+    await api.loadServerGroups();
+
     try {
       potentialPromise = api.runCommand(`${pluginName}.${commandName}`);
     } catch (e) {
diff --git a/src/load-plugins.js b/src/load-plugins.js
index babb1616..ab9e2693 100644
--- a/src/load-plugins.js
+++ b/src/load-plugins.js
@@ -9,6 +9,7 @@ import registerCommand from './commands';
 import { registerHook } from './hooks';
 import { registerPreparer } from './prepare-config';
 import { registerScrubber } from './scrub-config';
+import { registerServerSource } from './server-sources';
 import { registerSwarmOptions } from './swarm-options';
 import resolveFrom from 'resolve-from';
 
@@ -95,6 +96,11 @@ function registerPlugin(plugin) {
   if (plugin.module.swarmOptions) {
     registerSwarmOptions(plugin.module.swarmOptions);
   }
+  if (plugin.module.serverSources) {
+    for (const [type, config] of Object.entries(plugin.module.serverSources)) {
+      registerServerSource(type, config);
+    }
+  }
 }
 
 export function loadPlugins(plugins) {
diff --git a/src/plugin-api.js b/src/plugin-api.js
index b1cc239a..534dd82e 100644
--- a/src/plugin-api.js
+++ b/src/plugin-api.js
@@ -17,6 +17,7 @@ import path from 'path';
 import { runConfigPreps } from './prepare-config';
 import { scrubConfig } from './scrub-config';
 import serverInfo from './server-info';
+import { serverSources } from './server-sources';
 
 const { resolvePath, moduleNotFoundIsPath } = utils;
 const log = debug('mup:api');
@@ -29,6 +30,7 @@ export default class PluginAPI {
     this.config = null;
     this.settings = null;
     this.sessions = null;
+    this._serverGroupServers = Object.create(null);
     this._enabledSessions = program.servers ? program.servers.split(' ') : [];
     this.configPath = program.config ? resolvePath(program.config) : path.join(this.base, 'mup.js');
     this.settingsPath = program.settings;
@@ -127,9 +129,9 @@ export default class PluginAPI {
       );
       console.log('    http://meteor-up.com/docs');
       console.log('');
-    }
 
-    this.validationErrors = problems;
+      this.validationErrors = problems;
+    }
 
     return problems;
   }
@@ -374,6 +376,56 @@ export default class PluginAPI {
     this._cachedServerInfo = null;
   }
 
+  async loadServerGroups() {
+    const { servers } = this.getConfig(false);
+
+    if (typeof servers !== 'object' || servers === null) {
+      return;
+    }
+
+    const promises = Object.entries(servers)
+      .filter(([, serverConfig]) => serverConfig && typeof serverConfig.source === 'string')
+      .map(async ([name, serverConfig]) => {
+        const source = serverConfig.source;
+
+        if (!(source in serverSources)) {
+          throw new Error(`Unrecognized server source: ${source}. Available: ${Object.keys(serverSources)}`);
+        }
+
+        const list = await serverSources[source].load(name, serverConfig, this);
+        this._serverGroupServers[name] = list;
+
+        // TODO: handle errors. We should delay throwing the error until
+        // we need the sessions from this server group
+      });
+
+    await Promise.all(promises);
+  }
+
+  async updateServerGroups() {
+    const { servers } = this.getConfig();
+
+    if (typeof servers !== 'object' || servers === null) {
+      return;
+    }
+
+    const promises = Object.entries(servers)
+      .filter(([, serverConfig]) => serverConfig && typeof serverConfig.source === 'string')
+      .map(async ([name, serverConfig]) => {
+        const source = serverConfig.source;
+
+        if (!(source in serverSources)) {
+          throw new Error(`Unrecognized server source: ${source}. Available: ${Object.keys(serverSources)}`);
+        }
+
+        await serverSources[source].update(name, serverConfig, this);
+        const list = await serverSources[source].load(name, serverConfig, this);
+        this._serverGroupServers[name] = list;
+      });
+
+    await Promise.all(promises);
+  }
+
   getSessions(modules = []) {
     const sessions = this._pickSessions(modules);
 
@@ -413,7 +465,16 @@ export default class PluginAPI {
           continue;
         }
 
-        if (this.sessions[name]) {
+        if (!this.sessions[name]) {
+          continue;
+        }
+
+        if (Array.isArray(this.sessions[name])) {
+          // Is a server group. Add the members of the group.
+          this.sessions[name].forEach(memberName => {
+            sessions[memberName] = this.sessions[memberName];
+          });
+        } else {
           sessions[name] = this.sessions[name];
         }
       }
@@ -427,21 +488,7 @@ export default class PluginAPI {
 
     this.sessions = {};
 
-    // `mup.servers` contains login information for servers
-    // Use this information to create nodemiral sessions.
-    for (const name in config.servers) {
-      if (!config.servers.hasOwnProperty(name)) {
-        continue;
-      }
-
-      if (
-        this._enabledSessions.length > 0 &&
-        this._enabledSessions.indexOf(name) === -1
-      ) {
-        continue;
-      }
-
-      const info = config.servers[name];
+    function createNodemiralSession(name, info) {
       const auth = {
         username: info.username
       };
@@ -478,9 +525,38 @@ export default class PluginAPI {
         process.exit(1);
       }
 
-      const session = nodemiral.session(info.host, auth, opts);
+      return nodemiral.session(info.host, auth, opts);
+    }
+
+    // `mup.servers` contains login information for servers
+    // Use this information to create nodemiral sessions.
+    for (const name in config.servers) {
+      if (!config.servers.hasOwnProperty(name)) {
+        continue;
+      }
+
+      if (
+        this._enabledSessions.length > 0 &&
+        this._enabledSessions.indexOf(name) === -1
+      ) {
+        // TODO: if server group, check if any servers in group are enabled
+        continue;
+      }
+
+      const info = config.servers[name];
 
-      this.sessions[name] = session;
+      if (typeof info.source === 'string') {
+        const servers = this._serverGroupServers[name];
+        servers.forEach(server => {
+          const session = createNodemiralSession(
+            server.name, server
+          );
+          this.sessions[server.name] = session;
+        });
+        this.sessions[name] = servers.map(s => s.name);
+      } else {
+        this.sessions[name] = createNodemiralSession(name, info);
+      }
     }
   }
 
@@ -491,7 +567,9 @@ export default class PluginAPI {
     }
 
     Object.keys(this.sessions).forEach(key => {
-      this.sessions[key].close();
+      if (!Array.isArray(this.sessions[key])) {
+        this.sessions[key].close();
+      }
     });
   }
 
diff --git a/src/plugins/default/command-handlers.js b/src/plugins/default/command-handlers.js
index 5fdb91b4..85f0ed50 100644
--- a/src/plugins/default/command-handlers.js
+++ b/src/plugins/default/command-handlers.js
@@ -21,7 +21,10 @@ export function restart() {
   log('exec => mup restart');
 }
 
-export function setup(api) {
+export async function setup(api) {
+  log('exec => mup setup');
+  await api.updateServerGroups();
+
   process.on('exit', code => {
     if (code > 0) {
       return;
@@ -32,7 +35,6 @@ export function setup(api) {
     console.log('    mup deploy');
   });
 
-  log('exec => mup setup');
 
   return api.runCommand('docker.setup');
 }
diff --git a/src/plugins/default/index.js b/src/plugins/default/index.js
index afb30694..aeaaa0d5 100644
--- a/src/plugins/default/index.js
+++ b/src/plugins/default/index.js
@@ -1,8 +1,11 @@
 import * as _commands from './commands';
+import _serverSources from './server-groups/index.js';
 import traverse from 'traverse';
 
 export const commands = _commands;
 
+export const serverSources = _serverSources;
+
 export function scrubConfig(config) {
   if (config.servers) {
     // eslint-disable-next-line
diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
new file mode 100644
index 00000000..62b7ecd0
--- /dev/null
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -0,0 +1,134 @@
+import axios from 'axios';
+import { generateName } from './utils';
+
+export default class DigitalOcean {
+  constructor(groupName, groupConfig) {
+    this.name = groupName;
+    this.config = groupConfig;
+
+    const tagPrefix = groupConfig.__tagPrefix || 'mup-';
+    this.tag = `${tagPrefix}-${this.name}`;
+  }
+
+  async getServers() {
+    // TODO: implement pagination
+    const results = await this._request(
+      'get',
+      `droplets?tag_name=${this.tag}&per_page=200`
+    );
+
+    return results.data.droplets.map(droplet => ({
+      name: droplet.name,
+      host: droplet.networks.v4.find(n => n.type === 'public').ip_address,
+      username: 'root',
+      pem: this.config.pem,
+      privateIp: droplet.networks.v4.find(n => n.type === 'private').ip_address,
+      __droplet: droplet
+    }));
+  }
+
+  async compareServers() {
+    const servers = await this.getServers();
+    const good = [];
+    const wrong = [];
+
+    servers.forEach(server => {
+      const droplet = server.__droplet;
+
+      if (
+        droplet.size_slug !== this.config.size ||
+        droplet.region.slug !== this.config.region
+      ) {
+        wrong.push(server);
+      } else {
+        good.push(server);
+      }
+    });
+
+    return {
+      wrong,
+      good
+    };
+  }
+
+  async removeServers(servers) {
+    const promises = servers.map(server => this._request(
+      'delete',
+      `droplets/${server.__droplet.id}`
+    ));
+
+    await Promise.all(promises);
+  }
+
+  async createServers(count) {
+    const names = [];
+
+    while (names.length < count) {
+      names.push(generateName(this.name));
+    }
+
+    const data = {
+      names,
+      region: this.config.region,
+      size: this.config.size,
+
+      // TODO: pick image from API
+      image: 'ubuntu-20-04-x64',
+
+      // eslint-disable-next-line camelcase
+      ssh_keys: [
+        // TODO: Replace the fingerprint in the config with the path to the
+        // public key. Then mup can create the fingerprint, and add the ssh key
+        // to digital ocean if missing. This would allow each developer to have
+        // their own keys as long as during `mup setup` the other public keys
+        // are added to the server
+        this.config.sshKeyFingerprint
+      ],
+      monitoring: true,
+      tags: [
+        this.tag
+      ]
+    };
+
+    const result = await this._request(
+      'post',
+      'droplets',
+      data,
+    );
+
+    const ids = result.data.droplets.map(droplet => droplet.id);
+    await Promise.all(ids.map(id => this._waitForDropletActive(id)));
+  }
+
+  async _waitForDropletActive(id) {
+    const TEN_MINUTES = 1000 * 60 * 10;
+    const timeoutAt = Date.now() + TEN_MINUTES;
+
+    while (Date.now() < timeoutAt) {
+      const response = await this._request(
+        'get',
+        `droplets/${id}`
+      );
+      const status = response.data.droplet.status;
+
+      if (status === 'active') {
+        return;
+      }
+
+      await new Promise(resolve => setTimeout(resolve, 1000 * 10));
+    }
+
+    throw new Error(`Timed out waiting for droplet ${id} to become active`);
+  }
+
+  _request(method, path, data) {
+    return axios({
+      method,
+      url: `https://api.digitalocean.com/v2/${path}`,
+      data,
+      headers: {
+        Authorization: `Bearer ${this.config.token}`
+      }
+    });
+  }
+}
diff --git a/src/plugins/default/server-groups/index.js b/src/plugins/default/server-groups/index.js
new file mode 100644
index 00000000..810e0477
--- /dev/null
+++ b/src/plugins/default/server-groups/index.js
@@ -0,0 +1,47 @@
+import DigitalOcean from './digital-ocean';
+
+function createSourceConfig(SourceAPI) {
+  return {
+    async load(name, groupConfig) {
+      const api = new SourceAPI(name, groupConfig);
+
+      return api.getServers();
+    },
+    async update(name, groupConfig) {
+      const api = new SourceAPI(name, groupConfig);
+      const { wrong, good } = await api.compareServers();
+
+      const addCount = Math.max(0, groupConfig.count - good.length);
+      const goodRemoveCount = Math.max(0, good.length - groupConfig.count);
+
+      // If the region or type changed, all of the servers are wrong.
+      // We want to temporarily keep some of the wrong servers so they can
+      // handle requests until the new servers are ready
+      const min = Math.ceil(groupConfig.count / 2);
+      const tempCount = Math.min(wrong.length, min - groupConfig.count);
+
+      if (goodRemoveCount > 0) {
+        console.log(`=> Removing ${goodRemoveCount} servers for ${name}`);
+        await api.removeServers(good.slice(0, goodRemoveCount));
+        console.log(`=> Finished removing ${goodRemoveCount} servers for ${name}`);
+      }
+      if (wrong.length > 0) {
+        // TODO: don't delete tempCount until mup successfully exits
+        console.log(`=> Removing ${wrong.length} servers for ${name}`);
+        await api.removeServers(wrong);
+        console.log(`=> Finished removing ${wrong.length} servers for ${name}`);
+      }
+      if (addCount > 0) {
+        console.log(`=> Creating ${addCount} servers for ${name}`);
+        await api.createServers(addCount);
+        console.log(`=> Finished creating ${addCount} servers for ${name}`);
+      }
+    }
+  };
+}
+
+const serverSources = {
+  'digital-ocean': createSourceConfig(DigitalOcean)
+};
+
+export default serverSources;
diff --git a/src/plugins/default/server-groups/utils.js b/src/plugins/default/server-groups/utils.js
new file mode 100644
index 00000000..e025ee74
--- /dev/null
+++ b/src/plugins/default/server-groups/utils.js
@@ -0,0 +1,7 @@
+import crypto from 'crypto';
+
+export function generateName(groupName) {
+  const randomString = crypto.randomBytes(4).toString('hex');
+
+  return `mup-${groupName}-${randomString}`;
+}
diff --git a/src/server-sources.js b/src/server-sources.js
new file mode 100644
index 00000000..fb3522ed
--- /dev/null
+++ b/src/server-sources.js
@@ -0,0 +1,9 @@
+export const serverSources = Object.create(null);
+
+export function registerServerSource(type, { load, update } = {}) {
+  if (type in serverSources) {
+    throw new Error(`Duplicate server sources: ${type}`);
+  }
+
+  serverSources[type] = { load, update };
+}

From 3a2f28c4d812fcb4f42d153660a48d2c71d64076 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 22 Sep 2021 16:41:55 -0500
Subject: [PATCH 08/36] Improve "mup status" when no servers

---
 src/plugins/default/command-handlers.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/plugins/default/command-handlers.js b/src/plugins/default/command-handlers.js
index 85f0ed50..5572c1b4 100644
--- a/src/plugins/default/command-handlers.js
+++ b/src/plugins/default/command-handlers.js
@@ -201,6 +201,12 @@ export async function status(api) {
     lines.push(text);
   });
 
+  if (lines.length === 0) {
+    overallColor = 'gray';
+    lines.push(chalk.gray('   No servers listed in config'));
+  }
+
   console.log(chalk[overallColor]('=> Servers'));
   console.log(lines.join('\n'));
+  console.log('');
 }

From 26b6670ace8d37bf1022362c6766979528b10d84 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 22 Sep 2021 16:44:07 -0500
Subject: [PATCH 09/36] Weaken eslint rules

---
 .eslintrc.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.eslintrc.yml b/.eslintrc.yml
index 022b75c0..81937b24 100644
--- a/.eslintrc.yml
+++ b/.eslintrc.yml
@@ -39,9 +39,6 @@ parserOptions:
     templateStrings: true
     unicodeCodePointEscapes: true
 rules:
-  arrow-parens:
-  - 2
-  - as-needed
   array-bracket-spacing:
   - 2
   - never
@@ -58,7 +55,6 @@ rules:
   comma-style:
   - 2
   - last
-  complexity: [1, 9]
   computed-property-spacing:
   - 2
   - never
@@ -290,4 +286,4 @@ rules:
   newline-per-chained-call: 0
   class-methods-use-this: 0
   no-empty-function: 0
-  sort-imports: [2, { memberSyntaxSortOrder: ['none', 'all', 'multiple', 'single'], ignoreCase: true  }]
+  sort-imports: 0

From 26697e2e2b273cbefc596f64c0fc14f461827440 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 29 Sep 2021 16:56:02 -0500
Subject: [PATCH 10/36] Expand server groups

---
 src/plugin-api.js                       | 20 +++++++++++
 src/plugins/default/command-handlers.js | 13 +++++---
 src/plugins/docker/command-handlers.js  |  5 ++-
 src/plugins/meteor/command-handlers.js  | 44 +++++++++++++------------
 src/plugins/meteor/status.js            | 11 +++----
 src/plugins/proxy/command-handlers.js   |  4 +--
 src/plugins/proxy/utils.js              |  6 ++--
 7 files changed, 64 insertions(+), 39 deletions(-)

diff --git a/src/plugin-api.js b/src/plugin-api.js
index 534dd82e..f0f1b3c9 100644
--- a/src/plugin-api.js
+++ b/src/plugin-api.js
@@ -345,6 +345,26 @@ export default class PluginAPI {
     });
   }
 
+  expandServers(serversObj) {
+    let result = {};
+    const serverConfig = this.getConfig().servers;
+
+    Object.entries(serversObj).forEach(([key, config]) => {
+      if (key in this._serverGroupServers) {
+        this._serverGroupServers[key].forEach(server => {
+          result[server.name] = { server, config };
+        });
+      } else {
+        result[key] = {
+          server: serverConfig[key],
+          config
+        };
+      }
+    });
+
+    return result;
+  }
+
   async getServerInfo(selectedServers, collectors) {
     if (this._cachedServerInfo && !collectors) {
       return this._cachedServerInfo;
diff --git a/src/plugins/default/command-handlers.js b/src/plugins/default/command-handlers.js
index 5572c1b4..361278ff 100644
--- a/src/plugins/default/command-handlers.js
+++ b/src/plugins/default/command-handlers.js
@@ -49,6 +49,7 @@ export function stop() {
 
 export function ssh(api) {
   const servers = api.getConfig().servers;
+  const expandedServers = api.expandServers(servers);
   let serverOption = api.getArgs()[1];
 
   // Check how many sessions are enabled. Usually is all servers,
@@ -56,7 +57,7 @@ export function ssh(api) {
   const enabledSessions = api.getSessionsForServers(Object.keys(servers))
     .filter(session => session);
 
-  if (!(serverOption in servers)) {
+  if (!(serverOption in expandedServers)) {
     if (enabledSessions.length === 1) {
       const selectedHost = enabledSessions[0]._host;
       serverOption = Object.keys(servers).find(
@@ -64,14 +65,14 @@ export function ssh(api) {
       );
     } else {
       console.log('mup ssh <server>');
-      console.log('Available servers are:\n', Object.keys(servers).join('\n '));
+      console.log('Available servers are:\n', Object.keys(expandedServers).join('\n '));
       process.exitCode = 1;
 
       return;
     }
   }
 
-  const server = servers[serverOption];
+  const server = expandedServers[serverOption].server;
   const sshOptions = api._createSSHOptions(server);
 
   const conn = new Client();
@@ -153,12 +154,14 @@ function statusColor(
 }
 
 export async function status(api) {
-  const servers = Object.values(api.getConfig().servers || {});
+  const config = api.getConfig();
+  const allServers = Object.values(api.expandServers(config.servers || {}))
+    .map(({ server }) => server);
   const lines = [];
   let overallColor = 'green';
   const command = 'lsb_release -r -s || echo "false"; lsb_release -is; apt-get -v &> /dev/null && echo "true" || echo "false"; echo $BASH';
   const results = await map(
-    servers,
+    allServers,
     server => api.runSSHCommand(server, command),
     { concurrency: 2 }
   );
diff --git a/src/plugins/docker/command-handlers.js b/src/plugins/docker/command-handlers.js
index 327b2073..2d67d733 100644
--- a/src/plugins/docker/command-handlers.js
+++ b/src/plugins/docker/command-handlers.js
@@ -248,8 +248,11 @@ export async function status(api) {
     return;
   }
 
+  const allServers = Object.values(api.expandServers(config.servers || {}))
+    .map(({ server }) => server);
+
   const results = await map(
-    Object.values(config.servers),
+    allServers,
     server => api.runSSHCommand(server, 'sudo docker version --format "{{.Server.Version}}"'),
     { concurrency: 2 }
   );
diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js
index 6508d625..eb09b2cf 100644
--- a/src/plugins/meteor/command-handlers.js
+++ b/src/plugins/meteor/command-handlers.js
@@ -244,7 +244,6 @@ export async function push(api) {
 export function envconfig(api) {
   log('exec => mup meteor envconfig');
   const {
-    servers,
     app,
     proxy,
     privateDockerRegistry
@@ -288,15 +287,16 @@ export function envconfig(api) {
   }
 
   const startHostVars = {};
+  const expandedServers = api.expandServers(app.servers);
 
-  Object.keys(app.servers).forEach(serverName => {
-    const host = servers[serverName].host;
+  Object.values(expandedServers).forEach(({ server, config }) => {
+    const host = server.host;
     const vars = {};
-    if (app.servers[serverName].bind) {
-      vars.bind = app.servers[serverName].bind;
+    if (config.bind) {
+      vars.bind = config.bind;
     }
-    if (app.servers[serverName].env && app.servers[serverName].env.PORT) {
-      vars.port = app.servers[serverName].env.PORT;
+    if (config.env && config.env.PORT) {
+      vars.port = config.env.PORT;
     }
     startHostVars[host] = vars;
   });
@@ -325,20 +325,20 @@ export function envconfig(api) {
   const env = createEnv(app, api.getSettings());
   const hostVars = {};
 
-  Object.keys(app.servers).forEach(key => {
-    const host = servers[key].host;
-    if (app.servers[key].env) {
+  Object.values(expandedServers).forEach(({ server, config }) => {
+    const host = server.host;
+    if (config.env) {
       hostVars[host] = {
         env: {
-          ...app.servers[key].env,
+          ...config.env,
           // We treat the PORT specially and do not pass it to the container
           PORT: undefined
         }
       };
     }
-    if (app.servers[key].settings) {
+    if (config.settings) {
       const settings = JSON.stringify(api.getSettingsFromPath(
-        app.servers[key].settings));
+        config.settings));
 
       if (hostVars[host]) {
         hostVars[host].env.METEOR_SETTINGS = settings;
@@ -478,21 +478,21 @@ export async function restart(api) {
 
 export async function debugApp(api) {
   const {
-    servers,
     app
   } = api.getConfig();
   let serverOption = api.getArgs()[2];
+  let expandedServers = api.expandServers(app.servers);
 
   // Check how many sessions are enabled. Usually is all servers,
   // but can be reduced by the `--servers` option
   const enabledSessions = api.getSessions(['app'])
     .filter(session => session);
 
-  if (!(serverOption in app.servers)) {
+  if (!(serverOption in expandedServers)) {
     if (enabledSessions.length === 1) {
       const selectedHost = enabledSessions[0]._host;
-      serverOption = Object.keys(app.servers).find(
-        name => servers[name].host === selectedHost
+      serverOption = Object.keys(expandedServers).find(
+        name => expandedServers[name].server.host === selectedHost
       );
     } else {
       console.log('mup meteor debug <server>');
@@ -503,7 +503,7 @@ export async function debugApp(api) {
     }
   }
 
-  const server = servers[serverOption];
+  const server = expandedServers[serverOption].server;
   console.log(`Setting up to debug app running on ${serverOption}`);
 
   const {
@@ -626,10 +626,12 @@ export async function status(api) {
     StatusDisplay
   } = api.statusHelpers;
   const overview = api.getOptions().overview;
-  const servers = Object.keys(config.app.servers)
+  const expandedServers = api.expandServers(config.app.servers);
+  const servers = Object.keys(expandedServers)
     .map(key => ({
-      ...config.servers[key],
-      name: key
+      ...expandedServers[key].server,
+      name: key,
+      overrides: expandedServers[key].config
     }));
 
   const results = await map(
diff --git a/src/plugins/meteor/status.js b/src/plugins/meteor/status.js
index 2232f36a..efd4108f 100644
--- a/src/plugins/meteor/status.js
+++ b/src/plugins/meteor/status.js
@@ -103,11 +103,10 @@ async function checkUrlLocally(server, appConfig, port) {
 
 function getCheckAddress(server, appConfig) {
   if (
-    appConfig.servers &&
-    appConfig.servers[server.name] &&
-    appConfig.servers[server.name].bind
+    server.overrides &&
+    server.overrides.bind
   ) {
-    return appConfig.servers[server.name].bind;
+    return server.overrides.bind;
   }
 
   if (appConfig.docker && appConfig.docker.bind) {
@@ -118,8 +117,8 @@ function getCheckAddress(server, appConfig) {
 }
 
 export async function checkUrls(server, appConfig, api) {
-  const port = appConfig.servers[server.name].env ?
-    appConfig.servers[server.name].env.PORT :
+  const port = server.overrides.env?.PORT ?
+    server.overrides.env.PORT :
     appConfig.env.PORT;
 
   const [
diff --git a/src/plugins/proxy/command-handlers.js b/src/plugins/proxy/command-handlers.js
index 24a3b6b3..535163da 100644
--- a/src/plugins/proxy/command-handlers.js
+++ b/src/plugins/proxy/command-handlers.js
@@ -47,7 +47,6 @@ export function leLogs(api) {
 export function setup(api) {
   log('exec => mup proxy setup');
   const config = api.getConfig().proxy;
-  const serverConfig = api.getConfig().servers;
   const appConfig = api.getConfig().app;
   const appName = appConfig.name;
 
@@ -145,8 +144,7 @@ export function setup(api) {
   }
 
   const hostnames = getLoadBalancingHosts(
-    serverConfig,
-    Object.keys(appConfig.servers)
+    api.expandServers(appConfig.servers),
   );
 
   list.executeScript('Configure Nginx Upstream', {
diff --git a/src/plugins/proxy/utils.js b/src/plugins/proxy/utils.js
index 9ade7a99..fe582520 100644
--- a/src/plugins/proxy/utils.js
+++ b/src/plugins/proxy/utils.js
@@ -44,8 +44,8 @@ export function normalizeUrl(config, env) {
   return _config.app.env.ROOT_URL;
 }
 
-export function getLoadBalancingHosts(serverConfig, serverNames) {
-  return serverNames.map(name =>
-    serverConfig[name].privateIp || serverConfig[name].host
+export function getLoadBalancingHosts(expandedServers) {
+  return Object.values(expandedServers).map(({ server }) =>
+    server.privateIp || server.host
   );
 }

From 3a2d136f461d0e3a6f3f7c0b91878eec501f85a9 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 29 Sep 2021 16:56:57 -0500
Subject: [PATCH 11/36] Allow overriding commands

---
 src/load-plugins.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/load-plugins.js b/src/load-plugins.js
index ab9e2693..1d5471d6 100644
--- a/src/load-plugins.js
+++ b/src/load-plugins.js
@@ -73,7 +73,14 @@ export function locatePluginDir(name, configPath, appPath) {
 function registerPlugin(plugin) {
   if (plugin.module.commands) {
     Object.keys(plugin.module.commands).forEach(key => {
-      registerCommand(plugin.name, key, plugin.module.commands[key]);
+      let command = plugin.module.commands[key];
+      registerCommand(
+        // The __plugin option can be used to change the top-level command
+        // the command is added to
+        command.__plugin || plugin.name,
+        key,
+        plugin.module.commands[key]
+      );
     });
   }
   if (plugin.module.hooks) {

From 7c99c0613a462927605946c5520783bbafc03d35 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 30 Sep 2021 14:22:42 -0500
Subject: [PATCH 12/36] Wait for new servers to accept ssh connections

---
 .../default/server-groups/digital-ocean.js    | 13 +++++--
 src/plugins/default/server-groups/index.js    | 12 ++++---
 src/plugins/default/server-groups/utils.js    | 34 +++++++++++++++++++
 3 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
index 62b7ecd0..d8677f3a 100644
--- a/src/plugins/default/server-groups/digital-ocean.js
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -10,7 +10,7 @@ export default class DigitalOcean {
     this.tag = `${tagPrefix}-${this.name}`;
   }
 
-  async getServers() {
+  async getServers(ids) {
     // TODO: implement pagination
     const results = await this._request(
       'get',
@@ -24,7 +24,13 @@ export default class DigitalOcean {
       pem: this.config.pem,
       privateIp: droplet.networks.v4.find(n => n.type === 'private').ip_address,
       __droplet: droplet
-    }));
+    })).filter(server => {
+      if (ids) {
+        return ids.includes(server.__droplet.id);
+      }
+
+      return true;
+    });
   }
 
   async compareServers() {
@@ -98,6 +104,9 @@ export default class DigitalOcean {
 
     const ids = result.data.droplets.map(droplet => droplet.id);
     await Promise.all(ids.map(id => this._waitForDropletActive(id)));
+
+    return this.getServers(ids);
+  }
   }
 
   async _waitForDropletActive(id) {
diff --git a/src/plugins/default/server-groups/index.js b/src/plugins/default/server-groups/index.js
index 810e0477..dec65af1 100644
--- a/src/plugins/default/server-groups/index.js
+++ b/src/plugins/default/server-groups/index.js
@@ -1,14 +1,15 @@
 import DigitalOcean from './digital-ocean';
+import { waitForServers } from './utils';
 
 function createSourceConfig(SourceAPI) {
   return {
-    async load(name, groupConfig) {
-      const api = new SourceAPI(name, groupConfig);
+    async load(name, groupConfig, pluginApi) {
+      const api = new SourceAPI(name, groupConfig, pluginApi);
 
       return api.getServers();
     },
-    async update(name, groupConfig) {
-      const api = new SourceAPI(name, groupConfig);
+    async update(name, groupConfig, pluginApi) {
+      const api = new SourceAPI(name, groupConfig, pluginApi);
       const { wrong, good } = await api.compareServers();
 
       const addCount = Math.max(0, groupConfig.count - good.length);
@@ -33,7 +34,8 @@ function createSourceConfig(SourceAPI) {
       }
       if (addCount > 0) {
         console.log(`=> Creating ${addCount} servers for ${name}`);
-        await api.createServers(addCount);
+        const created = await api.createServers(addCount);
+        await waitForServers(created, pluginApi);
         console.log(`=> Finished creating ${addCount} servers for ${name}`);
       }
     }
diff --git a/src/plugins/default/server-groups/utils.js b/src/plugins/default/server-groups/utils.js
index e025ee74..2b84a48c 100644
--- a/src/plugins/default/server-groups/utils.js
+++ b/src/plugins/default/server-groups/utils.js
@@ -1,7 +1,41 @@
 import crypto from 'crypto';
+import { Client } from 'ssh2';
 
 export function generateName(groupName) {
   const randomString = crypto.randomBytes(4).toString('hex');
 
   return `mup-${groupName}-${randomString}`;
 }
+
+const FIVE_MINUTES = 1000 * 60 * 5;
+export function waitForServers(servers, api) {
+  async function waitForServer(server, startTime = Date.now()) {
+    if (Date.now() - startTime > FIVE_MINUTES) {
+      throw new Error('Timed out waiting for server to accept SSH connections');
+    }
+
+    try {
+      await new Promise((resolve, reject) => {
+        const ssh = api._createSSHOptions(server);
+        const conn = new Client();
+        conn.once('error', err => {
+          reject(err);
+        }).once('ready', () => {
+          conn.end();
+          resolve();
+        }).connect(ssh);
+      });
+    } catch (e) {
+      if (e.code !== 'ECONNREFUSED') {
+        console.dir(e);
+      }
+      await new Promise(resolve => setTimeout(resolve, 1000 * 5));
+
+      return waitForServer(server);
+    }
+  }
+
+  return Promise.all(
+    servers.map(server => waitForServer(server))
+  );
+}

From 86fbbc442a84868f42d2baa3eabfd5c8c8c8da9d Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 30 Sep 2021 14:23:06 -0500
Subject: [PATCH 13/36] Retry installing docker if it fails

---
 src/plugins/docker/assets/docker-setup.sh | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/plugins/docker/assets/docker-setup.sh b/src/plugins/docker/assets/docker-setup.sh
index 2833e5a8..dda20792 100644
--- a/src/plugins/docker/assets/docker-setup.sh
+++ b/src/plugins/docker/assets/docker-setup.sh
@@ -18,8 +18,17 @@ echo "Major" $majorVersion
 echo "Minor" $minorVersion
 set -e
 
-if [ ! "$hasDocker" ]; then
+retry_install () {
+  echo "waiting 30 seconds"
+  sleep 30
+  echo "trying installation again"
   install_docker
+}
+
+if [ ! "$hasDocker" ]; then
+  # If the server was just created, it might still be running some apt-get
+  # commands and installing docker could fail due to apt-get locks.
+  install_docker || retry_install || retry_install
 
 elif [ "$minimumMajor" -gt "$majorVersion" ]; then
   echo "major wrong"

From 7ab10a0e0fdac31a9de6cb952251546d6acbd2b5 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 30 Sep 2021 14:23:54 -0500
Subject: [PATCH 14/36] Combine stderr and stdout logs

---
 src/nodemiral.js                          | 19 ++++++++++++-------
 src/plugins/docker/assets/docker-setup.sh |  1 +
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/nodemiral.js b/src/nodemiral.js
index 13d71010..f714c458 100644
--- a/src/nodemiral.js
+++ b/src/nodemiral.js
@@ -63,13 +63,18 @@ function createCallback(cb, varsMapper) {
       return cb(err);
     }
     if (code > 0) {
-      const message = `
-      ------------------------------------STDERR------------------------------------
-      ${logs.stderr.substring(logs.stderr.length - 4200)}
-      ------------------------------------STDOUT------------------------------------
-      ${logs.stdout.substring(logs.stdout.length - 4200)}
-      ------------------------------------------------------------------------------
-      `;
+      let message = '';
+      if (!logs.stderr.length && logs.stdout.length) {
+        message = logs.stdout.substring(logs.stdout.length - 4200);
+      } else {
+        message = `
+        ------------------------------------STDERR------------------------------------
+        ${logs.stderr.substring(logs.stderr.length - 4200)}
+        ------------------------------------STDOUT------------------------------------
+        ${logs.stdout.substring(logs.stdout.length - 4200)}
+        ------------------------------------------------------------------------------
+        `;
+      }
 
       return cb(new Error(message));
     }
diff --git a/src/plugins/docker/assets/docker-setup.sh b/src/plugins/docker/assets/docker-setup.sh
index dda20792..2f82585d 100644
--- a/src/plugins/docker/assets/docker-setup.sh
+++ b/src/plugins/docker/assets/docker-setup.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+exec 2>&1
 
 # TODO make sure we can run docker in this server
 

From 8b3223ca158a5ebe1affabde24fa4c529eb9a41f Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 30 Sep 2021 14:25:25 -0500
Subject: [PATCH 15/36] Add ssh key to digital ocean

---
 .../default/server-groups/digital-ocean.js    | 55 ++++++++++++++++---
 src/plugins/default/server-groups/utils.js    | 17 +++++-
 2 files changed, 62 insertions(+), 10 deletions(-)

diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
index d8677f3a..a2cf1bdd 100644
--- a/src/plugins/default/server-groups/digital-ocean.js
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -1,11 +1,14 @@
 import axios from 'axios';
-import { generateName } from './utils';
+import { generateName, createFingerprint } from './utils';
+import fs from 'fs';
 
 export default class DigitalOcean {
-  constructor(groupName, groupConfig) {
+  constructor(groupName, groupConfig, pluginApi) {
     this.name = groupName;
     this.config = groupConfig;
 
+    this.publicKeyPath = pluginApi.resolvePath(this.config.sshKey.public);
+
     const tagPrefix = groupConfig.__tagPrefix || 'mup-';
     this.tag = `${tagPrefix}-${this.name}`;
   }
@@ -21,7 +24,7 @@ export default class DigitalOcean {
       name: droplet.name,
       host: droplet.networks.v4.find(n => n.type === 'public').ip_address,
       username: 'root',
-      pem: this.config.pem,
+      pem: this.config.sshKey.private,
       privateIp: droplet.networks.v4.find(n => n.type === 'private').ip_address,
       __droplet: droplet
     })).filter(server => {
@@ -67,6 +70,7 @@ export default class DigitalOcean {
   }
 
   async createServers(count) {
+    let fingerprint = await this._setupPublicKey();
     const names = [];
 
     while (names.length < count) {
@@ -83,12 +87,7 @@ export default class DigitalOcean {
 
       // eslint-disable-next-line camelcase
       ssh_keys: [
-        // TODO: Replace the fingerprint in the config with the path to the
-        // public key. Then mup can create the fingerprint, and add the ssh key
-        // to digital ocean if missing. This would allow each developer to have
-        // their own keys as long as during `mup setup` the other public keys
-        // are added to the server
-        this.config.sshKeyFingerprint
+        fingerprint
       ],
       monitoring: true,
       tags: [
@@ -107,6 +106,44 @@ export default class DigitalOcean {
 
     return this.getServers(ids);
   }
+
+  async _setupPublicKey() {
+    let content = fs.readFileSync(this.publicKeyPath, 'utf-8');
+    let fingerprint = createFingerprint(content);
+
+    try {
+      await this._request(
+        'get',
+        `account/keys/${fingerprint}`
+      );
+
+      return fingerprint;
+    } catch (e) {
+      if (!e || !e.response || !e.response.status === 404) {
+        console.dir(e);
+
+        throw e;
+      }
+
+      // Key doesn't exist. Ignore error since we will be adding it
+    }
+
+    try {
+      await this._request(
+        'post',
+        'account/keys',
+        {
+          // eslint-disable-next-line camelcase
+          public_key: content,
+          name: this.config.sshKey.name || this.name
+        }
+      );
+    } catch (e) {
+      console.dir(e);
+      throw e;
+    }
+
+    return fingerprint;
   }
 
   async _waitForDropletActive(id) {
diff --git a/src/plugins/default/server-groups/utils.js b/src/plugins/default/server-groups/utils.js
index 2b84a48c..2d925752 100644
--- a/src/plugins/default/server-groups/utils.js
+++ b/src/plugins/default/server-groups/utils.js
@@ -4,7 +4,22 @@ import { Client } from 'ssh2';
 export function generateName(groupName) {
   const randomString = crypto.randomBytes(4).toString('hex');
 
-  return `mup-${groupName}-${randomString}`;
+  return `${groupName}-${randomString}`;
+}
+
+export function createFingerprint(keyContent) {
+  keyContent = keyContent
+    // Remove key type at beginning
+    .replace(/(^ssh-[a-zA-Z0-9]*)/, '')
+    .trim()
+    // Remove comment at end
+    .replace(/ [^ ]+$/, '');
+
+  const buffer = Buffer.from(keyContent, 'base64');
+  const hash = crypto.createHash('md5').update(buffer).digest('hex');
+
+  // Add colons between every 2 characters
+  return hash.match(/.{1,2}/g).join(':');
 }
 
 const FIVE_MINUTES = 1000 * 60 * 5;

From eea28257a14c70a656e33fb929307524ab0a46dc Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 30 Sep 2021 14:26:13 -0500
Subject: [PATCH 16/36] Relesae 1.6.0-beta.1

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 971d7e00..4ac9e988 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mup",
-  "version": "1.5.4",
+  "version": "1.6.0-beta.1",
   "description": "Production Quality Meteor Deployments",
   "main": "lib/index.js",
   "repository": {

From 9d7c4e18ff55871bc4195516c542212f3358130e Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 13 Oct 2021 13:07:42 -0500
Subject: [PATCH 17/36] Increase combined logs length

---
 src/nodemiral.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nodemiral.js b/src/nodemiral.js
index f714c458..661aba8b 100644
--- a/src/nodemiral.js
+++ b/src/nodemiral.js
@@ -65,7 +65,7 @@ function createCallback(cb, varsMapper) {
     if (code > 0) {
       let message = '';
       if (!logs.stderr.length && logs.stdout.length) {
-        message = logs.stdout.substring(logs.stdout.length - 4200);
+        message = logs.stdout.substring(logs.stdout.length - 8400);
       } else {
         message = `
         ------------------------------------STDERR------------------------------------

From daf2041e38b3b7ed4d8b1e073d6923ab1becb431 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Thu, 14 Oct 2021 21:11:04 -0500
Subject: [PATCH 18/36] Version app bundles

---
 .eslintrc.yml                                 |   2 +-
 src/plugin-api.js                             |  11 +-
 src/plugins/meteor/assets/clean-versions.sh   |   8 +
 .../meteor/assets/meteor-deploy-check.sh      |  84 +++++---
 src/plugins/meteor/assets/meteor-start.sh     |  12 +-
 src/plugins/meteor/assets/prepare-bundle.sh   |   3 +-
 src/plugins/meteor/assets/templates/start.sh  |   9 +-
 src/plugins/meteor/command-handlers.js        | 201 ++++++++++++++----
 src/plugins/meteor/commands.js                |  14 ++
 src/plugins/meteor/rollback.js                | 103 +++++++++
 src/plugins/meteor/state.js                   |   7 +
 src/plugins/meteor/utils.js                   | 117 ++++++++--
 src/server-info.js                            |  12 +-
 13 files changed, 479 insertions(+), 104 deletions(-)
 create mode 100644 src/plugins/meteor/assets/clean-versions.sh
 create mode 100644 src/plugins/meteor/rollback.js
 create mode 100644 src/plugins/meteor/state.js

diff --git a/.eslintrc.yml b/.eslintrc.yml
index 81937b24..8a1e64ef 100644
--- a/.eslintrc.yml
+++ b/.eslintrc.yml
@@ -230,7 +230,7 @@ rules:
   - 2
   - after
   padded-blocks: [2, 'never']
-  prefer-const: 2
+  prefer-const: 0
   prefer-reflect: 0
   prefer-spread: 2
   quote-props:
diff --git a/src/plugin-api.js b/src/plugin-api.js
index f0f1b3c9..00b955cf 100644
--- a/src/plugin-api.js
+++ b/src/plugin-api.js
@@ -369,12 +369,12 @@ export default class PluginAPI {
     if (this._cachedServerInfo && !collectors) {
       return this._cachedServerInfo;
     }
-    const serverConfig = this.getConfig().servers;
+    const serverConfig = this.expandServers(this.getConfig().servers);
 
     const servers = (
-      selectedServers || Object.keys(this.getConfig().servers)
+      selectedServers || Object.keys(serverConfig)
     ).map(serverName => ({
-      ...serverConfig[serverName],
+      ...serverConfig[serverName].server,
       name: serverName
     }));
 
@@ -443,7 +443,10 @@ export default class PluginAPI {
         this._serverGroupServers[name] = list;
       });
 
-    await Promise.all(promises);
+    await Promise.all(promises).catch(e => {
+      console.dir(e);
+      throw e;
+    });
   }
 
   getSessions(modules = []) {
diff --git a/src/plugins/meteor/assets/clean-versions.sh b/src/plugins/meteor/assets/clean-versions.sh
new file mode 100644
index 00000000..ad655bc3
--- /dev/null
+++ b/src/plugins/meteor/assets/clean-versions.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+APP_NAME=<%= appName %>
+IMAGE=<%= imagePrefix %><%= appName.toLowerCase() %>
+
+<% for(version of versions) { %>
+  docker rmi $IMAGE:<%- version %>
+<% } %>
diff --git a/src/plugins/meteor/assets/meteor-deploy-check.sh b/src/plugins/meteor/assets/meteor-deploy-check.sh
index c1808853..749356d2 100644
--- a/src/plugins/meteor/assets/meteor-deploy-check.sh
+++ b/src/plugins/meteor/assets/meteor-deploy-check.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+exec 2>&1
 
 APPNAME=<%= appName %>
 APP_PATH=/opt/$APPNAME
@@ -15,39 +16,64 @@ cd $APP_PATH
 revert_app () {
   echo "=> Container status:"
   sudo docker inspect $APPNAME --format "restarted: {{.RestartCount}} times {{json .NetworkSettings}} {{json .State}}"
-  echo "=> Logs:" 1>&2
+  echo ""
+  echo ""
+  echo "=================================="
+  echo "=> App Logs:" 1>&2
   sudo docker logs --tail=100 $APPNAME 1>&2
 
-  <% if (privateRegistry) { %>
-    sudo docker pull $IMAGE:previous || true
+  <% if (canRollback) { %>
+    # Record that the version failed
+    FAILED_VERSION=$(tail -1 /opt/$APPNAME/config/version-history.txt) || ''
+    PREVIOUS_VERSION=$(tail -n 2 /opt/$APPNAME/config/version-history.txt | head -1) || ''
+
+    # Make sure we can roll back using a versioned image
+    if [ -n "$FAILED_VERSION" ] && [ -n "$PREVIOUS_VERSION" ]; then
+      <% if (recordFailed) { %>
+        # TODO: limit how large this file can grow
+        echo "$FAILED_VERSION" >> /opt/$APPNAME/config/failed-versions.txt
+      <% } %>
+
+      # Remove last line so we don't rollback to the failed version
+      head -n -1 /opt/$APPNAME/config/version-history.txt > /opt/$APPNAME/config/version-history.tmp.txt 
+      mv /opt/$APPNAME/config/version-history.tmp.txt /opt/$APPNAME/config/version-history.txt
+
+      sudo bash $START_SCRIPT > /dev/null 2>&1
+      echo " " 1>&2
+      echo "=> Redeploying previous version of the app" 1>&2
+      echo " " 1>&2
+
+    # If the versioned image isn't available, check if there is a previous image
+    elif sudo docker image inspect $IMAGE:latest >/dev/null; then
+      sudo docker tag $IMAGE:previous $IMAGE:latest
+
+      <% if (privateRegistry) { %>
+        sudo docker push $IMAGE:latest
+        docker image prune -f
+      <% } %>
+
+      echo "latest" > /opt/$APPNAME/config/version-history.txt
+      sudo bash $START_SCRIPT > /dev/null 2>&1
+
+      echo " " 1>&2
+      echo "=> Redeploying previous version of the app" 1>&2
+      echo " " 1>&2
+
+    elif [ -d last ]; then
+      rm /opt/$APPNAME/config/version-history.txt
+      sudo mv last current
+      sudo bash $START_SCRIPT > /dev/null 2>&1
+
+      echo " " 1>&2
+      echo "=> Redeploying previous version of the app" 1>&2
+      echo " " 1>&2
+    fi
   <% } %>
-  
-  if sudo docker image inspect $IMAGE:previous >/dev/null 2>&1; then
-    sudo docker tag $IMAGE:previous $IMAGE:latest
-
-    <% if (privateRegistry) { %>
-      sudo docker push $IMAGE:latest
-      docker image prune -f
-    <% } %>
-
-    sudo bash $START_SCRIPT > /dev/null 2>&1
-
-    echo " " 1>&2
-    echo "=> Redeploying previous version of the app" 1>&2
-    echo " " 1>&2
-
-  elif [ -d last ]; then
-    sudo mv last current
-    sudo bash $START_SCRIPT > /dev/null 2>&1
-
-    echo " " 1>&2
-    echo "=> Redeploying previous version of the app" 1>&2
-    echo " " 1>&2
-  fi
+}
 
-  echo
-  echo "To see more logs type 'mup logs --tail=200'"
-  echo ""
+finish () {
+  # TODO: if canRollback is true, remove version from failed list
+  exit 0
 }
 
 START_TIME=$(date +%s)
diff --git a/src/plugins/meteor/assets/meteor-start.sh b/src/plugins/meteor/assets/meteor-start.sh
index feb03243..46bb9798 100644
--- a/src/plugins/meteor/assets/meteor-start.sh
+++ b/src/plugins/meteor/assets/meteor-start.sh
@@ -6,6 +6,11 @@ APP_DIR=/opt/<%=appName %>
 IMAGE=mup-<%= appName.toLowerCase() %>
 PRIVATE_REGISTRY=<%- privateRegistry ? 0 : 1 %>
 
+<% if (typeof version === 'number') { %>
+  # TODO: limit how large this file can grow
+  echo "<%= version %>" >> $APP_DIR/config/version-history.txt
+<% } %>
+
 <% if (removeImage) { %>
 echo "Removing images"
 # Run when the docker image doesn't support prepare-bundle.sh.
@@ -13,13 +18,15 @@ echo "Removing images"
 # Removes the latest image, so the start script will use the bundle instead
 sudo docker rmi $IMAGE:latest || true
 sudo docker images
+rm $APP_DIR/config/version-history.txt || true
 <% } %>
 
 # save the last known version
 cd $APP_DIR
-if sudo docker image inspect $IMAGE:latest >/dev/null || [ "$PRIVATE_REGISTRY" == "0" ]; then
+if sudo docker image inspect $IMAGE:latest >/dev/null || [ "$PRIVATE_REGISTRY" == "0" ] || [ -f $APP_DIR/config/version-history.txt ]; then
   echo "using image"
   sudo rm -rf current || true
+
 else
   echo "using bundle"
   sudo rm -rf last
@@ -32,8 +39,7 @@ else
   sudo docker rmi $IMAGE:previous || true
 fi
 
-# TODO: clean up the last folder when the private registry has a previous image
-if sudo docker image inspect $IMAGE:previous >/dev/null; then
+if sudo docker image inspect $IMAGE:previous >/dev/null || [[ $(wc -l < $APP_DIR/config/version-history.txt) -ge 2 ]] ; then
   echo "removing last"
   sudo rm -rf last
 fi
diff --git a/src/plugins/meteor/assets/prepare-bundle.sh b/src/plugins/meteor/assets/prepare-bundle.sh
index 02852513..65efaf5d 100644
--- a/src/plugins/meteor/assets/prepare-bundle.sh
+++ b/src/plugins/meteor/assets/prepare-bundle.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+exec 2>&1
 
 set -e
 
@@ -85,7 +86,7 @@ sudo docker tag $IMAGE:build $IMAGE:<%= tag %>
   # Fails if the previous tag doesn't exist (such as during the initial deploy)
   sudo docker push $IMAGE:previous || true
 
-  sudo docker push $IMAGE:latest
+  sudo docker push $IMAGE:<%= tag %>
   
 <% } %>
 
diff --git a/src/plugins/meteor/assets/templates/start.sh b/src/plugins/meteor/assets/templates/start.sh
index d1c8f8b6..cebf20f2 100644
--- a/src/plugins/meteor/assets/templates/start.sh
+++ b/src/plugins/meteor/assets/templates/start.sh
@@ -10,14 +10,19 @@ BIND=<%= bind %>
 NGINX_PROXY_VERSION="1.0.0"
 LETS_ENCRYPT_VERSION="v1.13.1"
 APP_IMAGE=<%- imagePrefix %><%= appName.toLowerCase() %>
-IMAGE=$APP_IMAGE:latest
+
+TAG=$(tail -1 $APP_PATH/config/version-history.txt)
+TAG=${TAG:="latest"}
+echo "TAG: $TAG"
+
+IMAGE=$APP_IMAGE:$TAG
 VOLUME="--volume=$BUNDLE_PATH:/bundle"
 LOCAL_IMAGE=false
 
 <% if (!privateRegistry) { %>
 sudo docker image inspect $IMAGE >/dev/null || IMAGE=<%= docker.image %>
 
-if [ $IMAGE == $APP_IMAGE:latest  ]; then
+if [ $IMAGE == $APP_IMAGE:$TAG  ]; then
   VOLUME=""
   LOCAL_IMAGE=true
 fi
diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js
index eb09b2cf..97c10ad4 100644
--- a/src/plugins/meteor/command-handlers.js
+++ b/src/plugins/meteor/command-handlers.js
@@ -3,11 +3,11 @@ import {
   checkAppStarted,
   createEnv,
   createServiceConfig,
-  currentImageTag,
   escapeEnvQuotes,
   getImagePrefix,
   getNodeVersion,
   getSessions,
+  getVersions,
   shouldRebuild
 } from './utils';
 import buildApp, { archiveApp, cleanBuildDir } from './build.js';
@@ -16,6 +16,8 @@ import { map, promisify } from 'bluebird';
 import { prepareBundleLocally, prepareBundleSupported } from './prepare-bundle';
 import debug from 'debug';
 import nodemiral from '@zodern/nodemiral';
+import { rollback } from './rollback';
+import state from './state';
 
 
 const log = debug('mup:module:meteor');
@@ -142,51 +144,79 @@ export async function prepareBundle(api) {
   const buildOptions = appConfig.buildOptions;
   const bundlePath = api.resolvePath(buildOptions.buildLocation, 'bundle.tar.gz');
 
+  // await getVersions(api);
+
+  const sessions = api.getSessions(['app']);
+
+  const { latest, servers: serverVersions } = await getVersions(api);
+  const tag = latest + 1;
+  state.deployingVersion = tag;
+
   if (appConfig.docker.prepareBundleLocally) {
-    return prepareBundleLocally(buildOptions.buildLocation, bundlePath, api);
-  }
+    await prepareBundleLocally(buildOptions.buildLocation, bundlePath, api);
+  } else {
+    const list = nodemiral.taskList('Prepare App Bundle');
+    const nodeVersion = await getNodeVersion(bundlePath);
+
+    list.executeScript('Prepare Bundle', {
+      script: api.resolvePath(
+        __dirname,
+        'assets/prepare-bundle.sh'
+      ),
+      vars: {
+        appName: appConfig.name,
+        dockerImage: appConfig.docker.image,
+        env: escapeEnvQuotes(appConfig.env),
+        buildInstructions: appConfig.docker.buildInstructions || [],
+        nodeVersion,
+        stopApp: appConfig.docker.stopAppDuringPrepareBundle,
+        useBuildKit: appConfig.docker.useBuildKit,
+        tag,
+        privateRegistry: privateDockerRegistry,
+        imagePrefix: getImagePrefix(privateDockerRegistry)
+      }
+    });
 
-  const list = nodemiral.taskList('Prepare App Bundle');
+    // After running Prepare Bundle, the list of images will be out of date
+    api.serverInfoStale();
 
-  let tag = 'latest';
+    let prepareSessions = sessions;
+    if (privateDockerRegistry) {
+      prepareSessions = [sessions[0]].filter(s => s);
+    }
 
-  if (api.swarmEnabled()) {
-    const data = await api.getServerInfo();
-    tag = currentImageTag(data, appConfig.name) + 1;
+    await api.runTaskList(list, prepareSessions, {
+      series: true,
+      verbose: api.verbose
+    });
   }
 
-  const nodeVersion = await getNodeVersion(bundlePath);
+  const toClean = Object.create(null);
+
+  serverVersions.forEach(({ host, versions, current, previous }) => {
+    let toKeep = [current, previous, tag];
+    toClean[host] = {
+      versions: versions.filter(version => !toKeep.includes(version))
+    };
+  });
+
 
-  list.executeScript('Prepare Bundle', {
-    script: api.resolvePath(
-      __dirname,
-      'assets/prepare-bundle.sh'
-    ),
+  const list = nodemiral.taskList('Clean Up Versions');
+
+  list.executeScript('Clean up app versions', {
+    script: api.resolvePath(__dirname, 'assets/clean-versions.sh'),
     vars: {
+      // TODO: add a default version-history from other servers
+      // on servers that don't have a history so it has something to
+      // rollback to if the current deploy fails
       appName: appConfig.name,
-      dockerImage: appConfig.docker.image,
-      env: escapeEnvQuotes(appConfig.env),
-      buildInstructions: appConfig.docker.buildInstructions || [],
-      nodeVersion,
-      stopApp: appConfig.docker.stopAppDuringPrepareBundle,
-      useBuildKit: appConfig.docker.useBuildKit,
-      tag,
-      privateRegistry: privateDockerRegistry,
       imagePrefix: getImagePrefix(privateDockerRegistry)
-    }
+    },
+    hostVars: toClean
   });
 
-  // After running Prepare Bundle, the list of images will be out of date
-  api.serverInfoStale();
-
-  let sessions = api.getSessions(['app']);
-  if (privateDockerRegistry) {
-    sessions = sessions.slice(0, 1);
-  }
-
-  return api.runTaskList(list, sessions, {
-    series: true,
-    verbose: api.verbose
+  await api.runTaskList(list, sessions, {
+    series: false
   });
 }
 
@@ -368,7 +398,7 @@ export function envconfig(api) {
 
 export async function start(api) {
   log('exec => mup meteor start');
-  const config = api.getConfig().app;
+  const { app: config } = api.getConfig();
   const swarmEnabled = api.swarmEnabled();
 
   if (!config) {
@@ -376,12 +406,14 @@ export async function start(api) {
     process.exit(1);
   }
 
+  const isDeploy = api.commandHistory.find(entry =>
+    ['meteor.deploy', 'meteor.deployVersion'].includes(entry.name)
+  );
   const list = nodemiral.taskList('Start Meteor');
 
   if (swarmEnabled) {
     const currentService = await api.dockerServiceInfo(config.name);
-    const serverInfo = await api.getServerInfo();
-    const imageTag = currentImageTag(serverInfo, config.name);
+    const { latest: imageTag } = await getVersions(api);
 
     // TODO: make it work when the reverse proxy isn't enabled
     api.tasks.addCreateOrUpdateService(
@@ -390,16 +422,39 @@ export async function start(api) {
       currentService
     );
   } else {
-    addStartAppTask(list, api);
-    checkAppStarted(list, api);
+    addStartAppTask(list, api, { isDeploy, version: state.deployingVersion });
+    checkAppStarted(list, api, {
+      canRollback: isDeploy,
+      recordFailed: isDeploy && !api.commandHistory.find(
+        entry => entry.name === 'meteor.deployVersion'
+      )
+    });
   }
 
   const sessions = await getSessions(api);
 
-  return api.runTaskList(list, sessions, {
-    series: true,
-    verbose: api.verbose
-  });
+  try {
+    await api.runTaskList(list, sessions, {
+      series: true,
+      verbose: api.verbose
+    });
+
+    if (isDeploy) {
+      console.log(`Successfully deployed version ${state.deployingVersion}`);
+    }
+  } catch (e) {
+    if (
+      isDeploy &&
+      prepareBundleSupported(config.docker) &&
+      !api.swarmEnabled()
+    ) {
+      console.log('Deploy failed. Check the logs above for the reason');
+      console.log('=> Ensuring all servers have same version');
+      await rollback(api);
+    }
+
+    throw e;
+  }
 }
 
 export function deploy(api) {
@@ -419,6 +474,37 @@ export function deploy(api) {
     .then(() => api.runCommand('default.reconfig'));
 }
 
+export async function deployVersion(api) {
+  log('exec => mup meteor deploy');
+
+  // validate settings and config before starting
+  api.getSettings();
+  const config = api.getConfig().app;
+
+  if (!config) {
+    console.error('error: no configs found for meteor');
+    process.exit(1);
+  }
+
+  let version = api.getArgs()[2];
+
+  if (!version) {
+    console.error('Please provide a version');
+    process.exit(1);
+  }
+
+  version = parseInt(version, 10);
+
+  if (Number.isNaN(version)) {
+    console.log('Version is not a valid number');
+    process.exit(1);
+  }
+
+  state.deployingVersion = version;
+
+  return api.runCommand('default.reconfig');
+}
+
 export async function stop(api) {
   log('exec => mup meteor stop');
   const config = api.getConfig().app;
@@ -667,3 +753,32 @@ export async function status(api) {
 
   display.show(overview);
 }
+
+export async function listVersions(api) {
+  const versions = await getVersions(api);
+  console.log('Application versions:');
+  // TODO: when using private docker registry, combine versions
+  // and history to get a more complete list
+  versions.versions.forEach(version => {
+    let text = `  - ${version}`;
+
+    if (version === versions.current) {
+      text += ' (current)';
+    } else if (version === versions.previous) {
+      text += ' (previous)';
+    } else if (versions.failed.includes(version)) {
+      text += ' (failed)';
+    }
+
+    text = text.padEnd(17, ' ');
+
+    let date = versions.versionDates.get(version);
+    text += ` created ${date.toLocaleDateString('en-US', { dateStyle: 'short', timeStyle: 'short' })}`;
+
+    console.log(text);
+  });
+
+  console.log();
+  console.log('Switch to a different version by running:');
+  console.log('  mup meteor deploy-version <version>');
+}
diff --git a/src/plugins/meteor/commands.js b/src/plugins/meteor/commands.js
index 9dcacaf0..db0c64b3 100644
--- a/src/plugins/meteor/commands.js
+++ b/src/plugins/meteor/commands.js
@@ -87,6 +87,20 @@ export const debug = {
   handler: commandHandlers.debugApp
 };
 
+export const versions = {
+  description: 'List application versions',
+  handler: commandHandlers.listVersions
+};
+
+export const deployVersion = {
+  name: 'deploy-version [version]',
+  description: 'Deploy specific application version',
+  builder(yargs) {
+    yargs.strict(false);
+  },
+  handler: commandHandlers.deployVersion
+};
+
 // Hidden commands
 export const build = {
   description: false,
diff --git a/src/plugins/meteor/rollback.js b/src/plugins/meteor/rollback.js
new file mode 100644
index 00000000..98c877f6
--- /dev/null
+++ b/src/plugins/meteor/rollback.js
@@ -0,0 +1,103 @@
+import state from './state';
+import { addStartAppTask, checkAppStarted, getSessions, getVersions } from './utils';
+
+// After a failed deploy, ensure all servers are running the same version
+export async function rollback(api) {
+  const {
+    app: appConfig,
+    privateDockerRegistry
+  } = api.getConfig();
+  const sessions = await getSessions(api);
+  const versions = await getVersions(api);
+
+  if (versions.servers.length === 1) {
+    // There are no other servers to make sure it is consistent with
+    return;
+  }
+
+  // The servers that are running the new version, and are able to rollback
+  // to the previous version
+  const toRollback = versions.servers.filter(server => {
+    if (server.current !== state.deployingVersion) {
+      return false;
+    }
+
+    if (server.previous && privateDockerRegistry) {
+      return true;
+    }
+
+    // Make sure the server has the previous version
+    return server.versions.includes(server.previous);
+  });
+
+  function getSession(server) {
+    return sessions.find(session => session._host === server.host);
+  }
+
+  // TODO: when there two servers, make sure the app is available on the other
+  // server first. Otherwise, we might restart the only running instance of the
+  // app, causing downtime
+
+  // TODO: if all servers have the same current version, do nothing since
+  // it indicates the meteor-deploy-check script never never rolled back to
+  // the old version (we don't know if the app version is bad, or there was
+  // a connection issue or some other problem with the last server).
+
+  let list = new QuietList();
+  for (const server of toRollback) {
+    const session = getSession(server);
+    if (session) {
+      // Remove the failed version from history so we don't roll back to it
+      await api.runSSHCommand(
+        session,
+        [
+          `head -n -1 /opt/${appConfig.name}/config/version-history.txt > /opt/${appConfig.name}/config/version-history.tmp.txt`,
+          `mv /opt/${appConfig.name}/config/version-history.tmp.txt /opt/${appConfig.name}/config/version-history.txt`
+        ].join('; ')
+      );
+      let version = server.previous;
+      addStartAppTask(list, api, { isDeploy: false, version });
+      checkAppStarted(list, api);
+      try {
+        console.log(`  - ${server.name}: rolling back to previous version...`);
+        await list.run(session);
+        console.log(`  - ${server.name}: rolled back successfully`);
+      } catch (e) {
+        console.log(`  - ${server.name}: failed rolling back`);
+      }
+    }
+  }
+}
+
+class QuietList {
+  constructor() {
+    this.list = [];
+  }
+
+  executeScript(name, { script, vars }) {
+    this.list.push({ script, vars });
+  }
+
+  _run(session, script, vars) {
+    return new Promise((resolve, reject) => {
+      session.executeScript(script, {
+        vars
+      }, err => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+
+  async run(session) {
+    let list = this.list;
+    this.list = [];
+
+    for (const entry of list) {
+      await this._run(session, entry.script, entry.vars);
+    }
+  }
+}
diff --git a/src/plugins/meteor/state.js b/src/plugins/meteor/state.js
new file mode 100644
index 00000000..226691c8
--- /dev/null
+++ b/src/plugins/meteor/state.js
@@ -0,0 +1,7 @@
+// State shared between different commands
+
+const state = {
+  deployingVersion: null
+};
+
+export default state;
diff --git a/src/plugins/meteor/utils.js b/src/plugins/meteor/utils.js
index 81a78d43..1cfd18cc 100644
--- a/src/plugins/meteor/utils.js
+++ b/src/plugins/meteor/utils.js
@@ -1,4 +1,4 @@
-import { cloneDeep, flatMap } from 'lodash';
+import { cloneDeep } from 'lodash';
 import fs from 'fs';
 import os from 'os';
 import {
@@ -9,7 +9,7 @@ import { spawn } from 'child_process';
 import tar from 'tar';
 import uuid from 'uuid';
 
-export function checkAppStarted(list, api) {
+export function checkAppStarted(list, api, { canRollback, recordFailed } = {}) {
   const script = api.resolvePath(__dirname, 'assets/meteor-deploy-check.sh');
   const { app, privateDockerRegistry } = api.getConfig();
   const publishedPort = app.docker.imagePort || 80;
@@ -21,28 +21,28 @@ export function checkAppStarted(list, api) {
       appName: app.name,
       deployCheckPort: publishedPort,
       privateRegistry: privateDockerRegistry,
-      imagePrefix: getImagePrefix(privateDockerRegistry)
+      imagePrefix: getImagePrefix(privateDockerRegistry),
+      canRollback: canRollback || false,
+      recordFailed: recordFailed || false
     }
   });
 
   return list;
 }
 
-export function addStartAppTask(list, api) {
+export function addStartAppTask(list, api, { isDeploy, version } = {}) {
   const {
     app: appConfig,
     privateDockerRegistry
   } = api.getConfig();
-  const isDeploy = api.commandHistory.find(
-    ({ name }) => name === 'meteor.deploy'
-  );
 
   list.executeScript('Start Meteor', {
     script: api.resolvePath(__dirname, 'assets/meteor-start.sh'),
     vars: {
       appName: appConfig.name,
       removeImage: isDeploy && !prepareBundleSupported(appConfig.docker),
-      privateRegistry: privateDockerRegistry
+      privateRegistry: privateDockerRegistry,
+      version: typeof version === 'number' ? version : null
     }
   });
 
@@ -210,17 +210,96 @@ export function getImagePrefix(privateRegistry) {
   return 'mup-';
 }
 
-export function currentImageTag(serverInfo, appName) {
-  const result = flatMap(
-    Object.values(serverInfo),
-    ({images}) => images || []
-  )
-    .filter(image => image.Repository === `mup-${appName}`)
-    .map(image => parseInt(image.Tag, 10))
-    .filter(tag => !isNaN(tag))
-    .sort((a, b) => b - a);
-
-  return result[0] || 0;
+function mostCommon(values) {
+  let counts = new Map();
+  values.forEach(value => {
+    let number = counts.get(value) || 0;
+    counts.set(value, number + 1);
+  });
+
+  if (counts.size === 0) {
+    return null;
+  }
+
+  return [...counts].sort((a, b) => b[1] - a[1])[0][0];
+}
+
+export async function getVersions(api) {
+  const {
+    app: appConfig,
+    privateDockerRegistry
+  } = api.getConfig();
+
+  const imageName = `${getImagePrefix(privateDockerRegistry)}${appConfig.name.toLowerCase()}`;
+
+  const collector = {
+    images: {
+      command: `sudo docker images ${imageName} --format '{{json .}}'`,
+      parser: 'jsonArray'
+    },
+    history: {
+      command: `cat /opt/${appConfig.name}/config/version-history.txt`,
+      parser: 'text'
+    },
+    failed: {
+      command: `cat /opt/${appConfig.name}/config/failed-versions.txt`,
+      parser: 'text'
+    }
+  };
+
+  const data = await api.getServerInfo(
+    Object.keys(api.expandServers(appConfig.servers)),
+    collector
+  );
+  const result = {
+    latest: 0,
+    versions: [],
+    servers: [],
+    failed: [],
+    versionDates: new Map()
+  };
+
+  Object.values(data).forEach(entry => {
+    let serverVersions = [];
+
+    entry.images.forEach(image => {
+      let version = parseInt(image.Tag, 10);
+      if (!Number.isNaN(version)) {
+        serverVersions.push(version);
+      }
+
+      let date = new Date(image.CreatedAt);
+      let existingDate = result.versionDates.get(version);
+      if (!existingDate || existingDate.getTime() > date.getTime()) {
+        result.versionDates.set(version, date);
+      }
+    });
+
+    result.versions.push(...serverVersions);
+
+    const serverFailed = entry.failed ? entry.failed.split('\n').map(v => parseInt(v, 10)) : [];
+    result.failed.push(...serverFailed);
+
+    const history = entry.history ? entry.history.split('\n').map(v => parseInt(v, 10)) : [];
+
+    result.servers.push({
+      host: entry._host,
+      name: entry._serverName,
+      current: history[history.length - 1] || null,
+      previous: history[history.length - 2] || null,
+      versions: serverVersions.sort((a, b) => b - a),
+      history,
+      failed: serverFailed
+    });
+  });
+
+  result.versions = Array.from(new Set(result.versions)).sort((a, b) => b - a);
+  result.failed = Array.from(new Set(result.failed));
+  result.latest = result.versions[0] || 0;
+  result.current = mostCommon(result.servers.map(server => server.current));
+  result.previous = mostCommon(result.servers.map(server => server.previous));
+
+  return result;
 }
 
 export function readFileFromTar(tarPath, filePath) {
diff --git a/src/server-info.js b/src/server-info.js
index dbf904e1..b4a86b79 100644
--- a/src/server-info.js
+++ b/src/server-info.js
@@ -42,7 +42,14 @@ export const builtInParsers = {
 
     return null;
   },
-  jsonArray: parseJSONArray
+  jsonArray: parseJSONArray,
+  text(stdout, code) {
+    if (code === 0) {
+      return stdout.trim();
+    }
+
+    return null;
+  }
 };
 
 export const _collectors = {
@@ -78,8 +85,9 @@ function generateVarCommand(name, command) {
   return `
   echo "${prefix}${name}${suffix}"
   ${command} 2>&1
+  MUP_CODE=$?
   echo "${codeSeperator}"
-  echo $?
+  echo $MUP_CODE
   `;
 }
 

From 52a2312f33e41f0c6064d2e1cb8228ff7f171151 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Mon, 18 Oct 2021 10:23:51 -0500
Subject: [PATCH 19/36] Run setup if needed during reconfig/deploy

---
 src/check-setup.js                            | 90 +++++++++++++++++++
 src/load-plugins.js                           |  4 +
 src/plugin-api.js                             | 87 +++++++++++-------
 src/plugins/default/command-handlers.js       | 20 ++++-
 .../default/server-groups/digital-ocean.js    |  8 +-
 src/plugins/default/server-groups/index.js    | 10 ++-
 src/plugins/docker/command-handlers.js        |  2 +-
 src/plugins/docker/index.js                   | 26 ++++++
 src/plugins/meteor/index.js                   | 27 +++++-
 src/plugins/mongo/index.js                    | 29 ++++++
 src/plugins/proxy/command-handlers.js         |  2 +-
 src/plugins/proxy/index.js                    | 80 ++++++++++++++++-
 src/server-sources.js                         |  4 +-
 src/tasks/assets/check-setup.sh               | 18 ++++
 14 files changed, 363 insertions(+), 44 deletions(-)
 create mode 100644 src/check-setup.js
 create mode 100644 src/tasks/assets/check-setup.sh

diff --git a/src/check-setup.js b/src/check-setup.js
new file mode 100644
index 00000000..1edcbd58
--- /dev/null
+++ b/src/check-setup.js
@@ -0,0 +1,90 @@
+/* eslint-disable no-labels */
+const crypto = require('crypto');
+const fs = require('fs');
+
+const checkers = [];
+
+export function registerChecker(checker) {
+  checkers.push(checker);
+}
+
+function createKey(keyConfig = {}) {
+  const finalConfig = {
+    ...keyConfig
+  };
+  if (finalConfig.scripts) {
+    finalConfig.scripts = finalConfig.scripts.map(
+      path => fs.readFileSync(path, 'utf-8')
+    );
+  }
+
+  return crypto.createHash('sha256')
+    .update(JSON.stringify(finalConfig))
+    .digest('base64');
+}
+
+export async function checkSetup(pluginApi) {
+  const checks = await Promise.all(checkers.map(checker => checker(pluginApi)));
+
+  console.time('setup check');
+  const bySession = new Map();
+  checks.flat().forEach(check => {
+    let keyHash = createKey(check.setupKey);
+
+    check.sessions.forEach(session => {
+      const config = bySession.get(session) || {
+        keyHashes: {},
+        services: [],
+        containers: []
+      };
+
+      config.keyHashes[check.name] = keyHash;
+      config.services.push(...check.services || []);
+      config.containers.push(...check.containers || []);
+      bySession.set(session, config);
+    });
+  });
+
+  const promises = [];
+
+  bySession.forEach((config, session) => {
+    console.log(session._host, config);
+    const promise = new Promise(resolve => {
+      session.executeScript(
+        pluginApi.resolvePath(__dirname, './tasks/assets/check-setup.sh'),
+        {
+          vars: config
+        },
+        (err, code) => {
+          resolve(!err && code === 0);
+        }
+      );
+    });
+    promises.push(promise);
+  });
+
+  const result = await Promise.all(promises);
+  console.timeEnd('setup check');
+
+
+  return result.every(upToDate => upToDate);
+}
+
+export async function updateSetupKeys(api) {
+  const checks = await Promise.all(checkers.map(checker => checker(api)));
+
+  // TODO: parallelize this
+  for (const check of checks.flat()) {
+    const setupKeyHash = createKey(check.setupKey);
+
+    for (const session of check.sessions) {
+      // TODO: handle errors. Should retry, and after 3 tries give up
+      // Shouldn't throw since if the command fails mup will simply setup
+      // again next time
+      await api.runSSHCommand(
+        session,
+        `sudo mkdir -p /opt/.mup-setup && sudo echo "${setupKeyHash}" > /opt/.mup-setup/${check.name}.txt`
+      );
+    }
+  }
+}
diff --git a/src/load-plugins.js b/src/load-plugins.js
index 1d5471d6..c265757e 100644
--- a/src/load-plugins.js
+++ b/src/load-plugins.js
@@ -11,6 +11,7 @@ import { registerPreparer } from './prepare-config';
 import { registerScrubber } from './scrub-config';
 import { registerServerSource } from './server-sources';
 import { registerSwarmOptions } from './swarm-options';
+import { registerChecker } from './check-setup';
 import resolveFrom from 'resolve-from';
 
 const log = debug('mup:plugin-loader');
@@ -108,6 +109,9 @@ function registerPlugin(plugin) {
       registerServerSource(type, config);
     }
   }
+  if (plugin.module.checkSetup) {
+    registerChecker(plugin.module.checkSetup);
+  }
 }
 
 export function loadPlugins(plugins) {
diff --git a/src/plugin-api.js b/src/plugin-api.js
index 00b955cf..16fc7912 100644
--- a/src/plugin-api.js
+++ b/src/plugin-api.js
@@ -18,6 +18,7 @@ import { runConfigPreps } from './prepare-config';
 import { scrubConfig } from './scrub-config';
 import serverInfo from './server-info';
 import { serverSources } from './server-sources';
+import { checkSetup, updateSetupKeys } from './check-setup';
 
 const { resolvePath, moduleNotFoundIsPath } = utils;
 const log = debug('mup:api');
@@ -337,12 +338,17 @@ export default class PluginAPI {
       this._commandErrorHandler(e);
     }
 
-    await this._runPostHooks(name).then(() => {
-      // The post hooks for the first command should be the last thing run
-      if (firstCommand) {
-        this._cleanupSessions();
-      }
-    });
+    await this._runPostHooks(name);
+
+    if (name === 'default.setup') {
+      console.log('=> Storing setup config on servers');
+      await updateSetupKeys(this);
+    }
+
+    // The post hooks for the first command should be the last thing run
+    if (firstCommand) {
+      this._cleanupSessions();
+    }
   }
 
   expandServers(serversObj) {
@@ -396,14 +402,23 @@ export default class PluginAPI {
     this._cachedServerInfo = null;
   }
 
-  async loadServerGroups() {
+  async checkSetupNeeded() {
+    const [upToDate, serverGroupsUpToDate] = await Promise.all([
+      checkSetup(this),
+      this._serverGroupsUpToDate()
+    ]);
+
+    return !upToDate || !serverGroupsUpToDate;
+  }
+
+  _mapServerGroup(cb) {
     const { servers } = this.getConfig(false);
 
     if (typeof servers !== 'object' || servers === null) {
-      return;
+      return [];
     }
 
-    const promises = Object.entries(servers)
+    return Object.entries(servers)
       .filter(([, serverConfig]) => serverConfig && typeof serverConfig.source === 'string')
       .map(async ([name, serverConfig]) => {
         const source = serverConfig.source;
@@ -412,36 +427,48 @@ export default class PluginAPI {
           throw new Error(`Unrecognized server source: ${source}. Available: ${Object.keys(serverSources)}`);
         }
 
-        const list = await serverSources[source].load(name, serverConfig, this);
-        this._serverGroupServers[name] = list;
-
-        // TODO: handle errors. We should delay throwing the error until
-        // we need the sessions from this server group
+        return cb(name, serverConfig);
       });
+  }
+
+  async loadServerGroups() {
+    const promises = this._mapServerGroup(async (name, groupConfig) => {
+      const source = groupConfig.source;
+      const list = await serverSources[source].load(
+        { name, groupConfig }, this
+      );
+      this._serverGroupServers[name] = list;
+
+      // TODO: handle errors. We should delay throwing the error until
+      // we need the sessions from this server group
+    });
 
     await Promise.all(promises);
   }
 
-  async updateServerGroups() {
-    const { servers } = this.getConfig();
+  async _serverGroupsUpToDate() {
+    const promises = this._mapServerGroup((name, groupConfig) => {
+      const source = groupConfig.source;
 
-    if (typeof servers !== 'object' || servers === null) {
-      return;
-    }
+      return serverSources[source].upToDate({ name, groupConfig }, this);
+    });
 
-    const promises = Object.entries(servers)
-      .filter(([, serverConfig]) => serverConfig && typeof serverConfig.source === 'string')
-      .map(async ([name, serverConfig]) => {
-        const source = serverConfig.source;
+    const result = await Promise.all(promises);
 
-        if (!(source in serverSources)) {
-          throw new Error(`Unrecognized server source: ${source}. Available: ${Object.keys(serverSources)}`);
-        }
+    return result.every(upToDate => upToDate);
+  }
 
-        await serverSources[source].update(name, serverConfig, this);
-        const list = await serverSources[source].load(name, serverConfig, this);
-        this._serverGroupServers[name] = list;
-      });
+  async updateServerGroups() {
+    this.sessions = null;
+
+    const promises = this._mapServerGroup(async (name, groupConfig) => {
+      const source = groupConfig.source;
+      await serverSources[source].update({ name, groupConfig }, this);
+      const list = await serverSources[source].load(
+        { name, groupConfig }, this
+      );
+      this._serverGroupServers[name] = list;
+    });
 
     await Promise.all(promises).catch(e => {
       console.dir(e);
diff --git a/src/plugins/default/command-handlers.js b/src/plugins/default/command-handlers.js
index 361278ff..4187b3a4 100644
--- a/src/plugins/default/command-handlers.js
+++ b/src/plugins/default/command-handlers.js
@@ -5,16 +5,32 @@ import { map } from 'bluebird';
 
 const log = debug('mup:module:default');
 
-export function deploy() {
+export async function deploy(api) {
   log('exec => mup deploy');
+  console.log('=> Checking if server setup needed');
+  if (await api.checkSetupNeeded()) {
+    await api.runCommand('default.setup');
+  }
+
 }
 
 export function logs() {
   log('exec => mup logs');
 }
 
-export function reconfig() {
+export async function reconfig(api) {
   log('exec => mup reconfig');
+
+  if (api.commandHistory.find(({ name }) => name === 'default.deploy')) {
+    // We've already checked if setup is needed
+    return;
+  }
+
+  console.log('=> Checking if server setup needed');
+  if (await api.checkSetupNeeded()) {
+    await api.runCommand('default.setup');
+  }
+
 }
 
 export function restart() {
diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
index a2cf1bdd..0b0db7a2 100644
--- a/src/plugins/default/server-groups/digital-ocean.js
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -36,8 +36,12 @@ export default class DigitalOcean {
     });
   }
 
-  async compareServers() {
-    const servers = await this.getServers();
+  async compareServers(servers) {
+    if (!servers) {
+      // eslint-disable-next-line no-param-reassign
+      servers = await this.getServers();
+    }
+
     const good = [];
     const wrong = [];
 
diff --git a/src/plugins/default/server-groups/index.js b/src/plugins/default/server-groups/index.js
index dec65af1..52218f42 100644
--- a/src/plugins/default/server-groups/index.js
+++ b/src/plugins/default/server-groups/index.js
@@ -3,12 +3,18 @@ import { waitForServers } from './utils';
 
 function createSourceConfig(SourceAPI) {
   return {
-    async load(name, groupConfig, pluginApi) {
+    async load({ name, groupConfig }, pluginApi) {
       const api = new SourceAPI(name, groupConfig, pluginApi);
 
       return api.getServers();
     },
-    async update(name, groupConfig, pluginApi) {
+    async upToDate({ name, groupConfig, list }, pluginApi) {
+      const api = new SourceAPI(name, groupConfig, pluginApi);
+      const { wrong, good } = await api.compareServers(list);
+
+      return wrong.length === 0 && good.length === groupConfig.count;
+    },
+    async update({ name, groupConfig }, pluginApi) {
       const api = new SourceAPI(name, groupConfig, pluginApi);
       const { wrong, good } = await api.compareServers();
 
diff --git a/src/plugins/docker/command-handlers.js b/src/plugins/docker/command-handlers.js
index 2d67d733..825047e1 100644
--- a/src/plugins/docker/command-handlers.js
+++ b/src/plugins/docker/command-handlers.js
@@ -21,7 +21,7 @@ import nodemiral from '@zodern/nodemiral';
 
 const log = debug('mup:module:docker');
 
-function uniqueSessions(api) {
+export function uniqueSessions(api) {
   const { servers } = api.getConfig();
   const sessions = api.getSessions(['app', 'mongo', 'proxy']);
 
diff --git a/src/plugins/docker/index.js b/src/plugins/docker/index.js
index d82e359c..09b7060e 100644
--- a/src/plugins/docker/index.js
+++ b/src/plugins/docker/index.js
@@ -1,3 +1,4 @@
+import { uniqueSessions } from './command-handlers';
 import * as _commands from './commands';
 import { validateRegistry, validateSwarm } from './validate';
 
@@ -52,3 +53,28 @@ export function scrubConfig(config) {
 
   return config;
 }
+
+export async function checkSetup(api) {
+  const sessions = await uniqueSessions(api);
+  const config = api.getConfig();
+
+  return [
+    {
+      sessions,
+      name: 'docker',
+      setupKey: {
+        scripts: [
+          api.resolvePath(__dirname, 'assets/docker-setup.sh'),
+          api.resolvePath(__dirname, 'assets/install-docker.sh')
+        ],
+        config: {
+          privateDockerRegistry: config.privateDockerRegistry,
+          // TODO: fix this to avoid always needing to setup everytime when
+          // swarm is enabled
+          swarm: api.swarmEnabled() ? Date.now() : false
+        }
+      },
+      services: ['docker']
+    }
+  ];
+}
diff --git a/src/plugins/meteor/index.js b/src/plugins/meteor/index.js
index dc59e1a8..96507e10 100644
--- a/src/plugins/meteor/index.js
+++ b/src/plugins/meteor/index.js
@@ -1,7 +1,7 @@
 import * as _commands from './commands';
 import _validator from './validate';
 import { defaultsDeep } from 'lodash';
-import { tmpBuildPath } from './utils';
+import { getSessions, tmpBuildPath } from './utils';
 import traverse from 'traverse';
 
 export const description = 'Deploy and manage meteor apps';
@@ -138,3 +138,28 @@ export function swarmOptions(config) {
     };
   }
 }
+
+export async function checkSetup(api) {
+  const config = api.getConfig();
+  if (!config.app || config.app.type !== 'meteor') {
+    return [];
+  }
+
+  const sessions = await getSessions(api);
+
+  return [
+    {
+      sessions,
+      name: `meteor-${config.app.name}`,
+      setupKey: {
+        // TODO: handle legacy ssl configuration
+        scripts: [
+          api.resolvePath(__dirname, 'assets/meteor-setup.sh')
+        ],
+        config: {
+          name: config.app.name
+        }
+      }
+    }
+  ];
+}
diff --git a/src/plugins/mongo/index.js b/src/plugins/mongo/index.js
index db07c2dd..9cc2a592 100644
--- a/src/plugins/mongo/index.js
+++ b/src/plugins/mongo/index.js
@@ -47,3 +47,32 @@ export const hooks = {
     }
   }
 };
+
+export async function checkSetup(api) {
+  const config = api.getConfig();
+  if (!config.mongo) {
+    return [];
+  }
+
+  const sessions = api.getSessions(['mongo']);
+
+  return [
+    {
+      sessions,
+      name: `mongo-${config.app.name}`,
+      setupKey: {
+        scripts: [
+          api.resolvePath(__dirname, 'assets/mongo-setup.sh'),
+          api.resolvePath(__dirname, 'assets/templates/start.sh'),
+          api.resolvePath(__dirname, 'assets/mongo-start.sh')
+        ],
+        config: {
+          version: config.mongo.version
+        }
+      },
+      containers: [
+        'mongodb'
+      ]
+    }
+  ];
+}
diff --git a/src/plugins/proxy/command-handlers.js b/src/plugins/proxy/command-handlers.js
index 535163da..fd179fc5 100644
--- a/src/plugins/proxy/command-handlers.js
+++ b/src/plugins/proxy/command-handlers.js
@@ -6,7 +6,7 @@ import fs from 'fs';
 import nodemiral from '@zodern/nodemiral';
 
 const log = debug('mup:module:proxy');
-const PROXY_CONTAINER_NAME = 'mup-nginx-proxy';
+export const PROXY_CONTAINER_NAME = 'mup-nginx-proxy';
 
 export function logs(api) {
   log('exec => mup proxy logs');
diff --git a/src/plugins/proxy/index.js b/src/plugins/proxy/index.js
index cf8d93d8..b7617432 100644
--- a/src/plugins/proxy/index.js
+++ b/src/plugins/proxy/index.js
@@ -1,6 +1,6 @@
 import * as _commands from './commands';
-import { addProxyEnv, normalizeUrl } from './utils';
-import { updateProxyForLoadBalancing } from './command-handlers';
+import { addProxyEnv, getLoadBalancingHosts, getSessions, normalizeUrl } from './utils';
+import { PROXY_CONTAINER_NAME, updateProxyForLoadBalancing } from './command-handlers';
 import validator from './validate';
 
 export const description = 'Setup and manage reverse proxy and ssl';
@@ -37,7 +37,7 @@ export function prepareConfig(config) {
   }
 
   config.app.env.HTTP_FORWARDED_COUNT =
-  config.app.env.HTTP_FORWARDED_COUNT || 1;
+    config.app.env.HTTP_FORWARDED_COUNT || 1;
 
   if (swarmEnabled) {
     config.app.docker.networks = config.app.docker.networks || [];
@@ -135,3 +135,77 @@ export function swarmOptions(config) {
     };
   }
 }
+
+export async function checkSetup(api) {
+  const config = api.getConfig();
+  if (!config.proxy) {
+    return [];
+  }
+
+  const sessions = getSessions(api);
+
+  let configPaths = [];
+
+  if (config.proxy.nginxServerConfig) {
+    configPaths.push(
+      api.resolvePath(api.getBasePath(), config.proxy.nginxServerConfig)
+    );
+  }
+  if (config.proxy.nginxLocationConfig) {
+    configPaths.push(
+      api.resolvePath(api.getBasePath(), config.proxy.nginxLocationConfig)
+    );
+  }
+
+  if (config.proxy.ssl && config.proxy.ssl.crt) {
+    configPaths.push(
+      api.resolvePath(api.getBasePath(), config.proxy.ssl.crt)
+    );
+    configPaths.push(
+      api.resolvePath(api.getBasePath(), config.proxy.ssl.key)
+    );
+  }
+
+  let upstream = [];
+
+  if (config.loadBalancing) {
+    console.dir(config.app);
+    upstream = getLoadBalancingHosts(
+      api.expandServers(config.app.servers)
+    );
+  }
+
+  return [
+    {
+      sessions,
+      name: `proxy-${config.app.name}`,
+      setupKey: {
+        // TODO: handle legacy ssl configuration
+        scripts: [
+          api.resolvePath(__dirname, 'assets/proxy-setup.sh'),
+          api.resolvePath(__dirname, 'assets/templates/start.sh'),
+          api.resolvePath(__dirname, 'assets/nginx.tmpl'),
+          api.resolvePath(__dirname, 'assets/nginx-config.sh'),
+          api.resolvePath(__dirname, 'assets/ssl-cleanup.sh'),
+          api.resolvePath(__dirname, 'assets/ssl-setup.sh'),
+          api.resolvePath(__dirname, 'assets/upstream.sh'),
+          api.resolvePath(__dirname, 'assets/proxy-start.sh')
+        ],
+        config: {
+          domains: config.proxy.domains,
+          app: config.app.name,
+          letsEncryptEmail: config.ssl ? config.ssl.letsEncryptEmail : '',
+          swarm: api.swarmEnabled(),
+          clientUploadLimit: config.clientUploadLimit,
+          upstream,
+          stickySessions: config.stickySessions,
+          appPort: config.app.env.PORT
+        }
+      },
+      containers: [
+        `${PROXY_CONTAINER_NAME}-letsencrypt`,
+        PROXY_CONTAINER_NAME
+      ]
+    }
+  ];
+}
diff --git a/src/server-sources.js b/src/server-sources.js
index fb3522ed..8d624b3a 100644
--- a/src/server-sources.js
+++ b/src/server-sources.js
@@ -1,9 +1,9 @@
 export const serverSources = Object.create(null);
 
-export function registerServerSource(type, { load, update } = {}) {
+export function registerServerSource(type, { load, upToDate, update } = {}) {
   if (type in serverSources) {
     throw new Error(`Duplicate server sources: ${type}`);
   }
 
-  serverSources[type] = { load, update };
+  serverSources[type] = { load, upToDate, update };
 }
diff --git a/src/tasks/assets/check-setup.sh b/src/tasks/assets/check-setup.sh
new file mode 100644
index 00000000..c1e81874
--- /dev/null
+++ b/src/tasks/assets/check-setup.sh
@@ -0,0 +1,18 @@
+set -e
+
+<% for(const [ name, value ] of Object.entries(keyHashes)) { %>
+if [ $(sudo cat /opt/.mup-setup/<%- name %>.txt) != "<%- value %>" ]; then
+  exit 1
+fi
+<% } %>
+
+<% for(const service of services) { %>
+sudo service <%- service %> status
+<% } %>
+
+<% for (const container of containers) { %>
+STATUS="$(sudo docker inspect --format='{{.State.Running}}' <%- container %> 2> /dev/null)"
+if [ "$STATUS" == 'false' ] || [ -z "$STATUS" ]; then
+  exit 1
+fi
+<% } %>

From 1f26709fd9dab0700f26154fbe8ebf6ce8a73ec3 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Mon, 18 Oct 2021 10:38:00 -0500
Subject: [PATCH 20/36] Fix lint errors

---
 src/plugins/default/command-handlers.js    | 2 --
 src/plugins/default/server-groups/index.js | 5 +++--
 src/plugins/default/server-groups/utils.js | 4 ++--
 src/plugins/meteor/status.js               | 2 +-
 4 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/plugins/default/command-handlers.js b/src/plugins/default/command-handlers.js
index 4187b3a4..4cd5a1ee 100644
--- a/src/plugins/default/command-handlers.js
+++ b/src/plugins/default/command-handlers.js
@@ -11,7 +11,6 @@ export async function deploy(api) {
   if (await api.checkSetupNeeded()) {
     await api.runCommand('default.setup');
   }
-
 }
 
 export function logs() {
@@ -30,7 +29,6 @@ export async function reconfig(api) {
   if (await api.checkSetupNeeded()) {
     await api.runCommand('default.setup');
   }
-
 }
 
 export function restart() {
diff --git a/src/plugins/default/server-groups/index.js b/src/plugins/default/server-groups/index.js
index 52218f42..892f0537 100644
--- a/src/plugins/default/server-groups/index.js
+++ b/src/plugins/default/server-groups/index.js
@@ -21,11 +21,12 @@ function createSourceConfig(SourceAPI) {
       const addCount = Math.max(0, groupConfig.count - good.length);
       const goodRemoveCount = Math.max(0, good.length - groupConfig.count);
 
+      // TODO: finish implementing this
       // If the region or type changed, all of the servers are wrong.
       // We want to temporarily keep some of the wrong servers so they can
       // handle requests until the new servers are ready
-      const min = Math.ceil(groupConfig.count / 2);
-      const tempCount = Math.min(wrong.length, min - groupConfig.count);
+      // const min = Math.ceil(groupConfig.count / 2);
+      // const tempCount = Math.min(wrong.length, min - groupConfig.count);
 
       if (goodRemoveCount > 0) {
         console.log(`=> Removing ${goodRemoveCount} servers for ${name}`);
diff --git a/src/plugins/default/server-groups/utils.js b/src/plugins/default/server-groups/utils.js
index 2d925752..6f4df62d 100644
--- a/src/plugins/default/server-groups/utils.js
+++ b/src/plugins/default/server-groups/utils.js
@@ -8,14 +8,14 @@ export function generateName(groupName) {
 }
 
 export function createFingerprint(keyContent) {
-  keyContent = keyContent
+  const cleanedContent = keyContent
     // Remove key type at beginning
     .replace(/(^ssh-[a-zA-Z0-9]*)/, '')
     .trim()
     // Remove comment at end
     .replace(/ [^ ]+$/, '');
 
-  const buffer = Buffer.from(keyContent, 'base64');
+  const buffer = Buffer.from(cleanedContent, 'base64');
   const hash = crypto.createHash('md5').update(buffer).digest('hex');
 
   // Add colons between every 2 characters
diff --git a/src/plugins/meteor/status.js b/src/plugins/meteor/status.js
index efd4108f..01eea1c5 100644
--- a/src/plugins/meteor/status.js
+++ b/src/plugins/meteor/status.js
@@ -117,7 +117,7 @@ function getCheckAddress(server, appConfig) {
 }
 
 export async function checkUrls(server, appConfig, api) {
-  const port = server.overrides.env?.PORT ?
+  const port = server.overrides.env && server.overrides.env.PORT ?
     server.overrides.env.PORT :
     appConfig.env.PORT;
 

From 33e7bb1ef951106ec65514c8d39663e7c0b0b116 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Mon, 18 Oct 2021 10:41:11 -0500
Subject: [PATCH 21/36] Remove enableUploadProgressBar from example config

---
 docs/docs.md | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/docs/docs.md b/docs/docs.md
index 0a96ed90..435dccef 100644
--- a/docs/docs.md
+++ b/docs/docs.md
@@ -236,11 +236,7 @@ module.exports = {
     // lets you define which port to check after the deploy process, if it
     // differs from the meteor port you are serving
     // (like meteor behind a proxy/firewall) (optional)
-    deployCheckPort: 80,
-
-    // Shows progress bar while uploading bundle to server
-    // Since Meteor Up 1.6, this option does nothing, and is always true.
-    enableUploadProgressBar: true
+    deployCheckPort: 80
   },
 
   // (optional) Use built-in mongodb. Remove it to use a remote MongoDB

From d2fe33e5938806d9a3ed523f25005ade0a4e9f1f Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Mon, 18 Oct 2021 10:42:46 -0500
Subject: [PATCH 22/36] Fix unit tests

---
 src/__tests__/plugin-api.unit.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/__tests__/plugin-api.unit.js b/src/__tests__/plugin-api.unit.js
index 5234a3c1..187660a5 100644
--- a/src/__tests__/plugin-api.unit.js
+++ b/src/__tests__/plugin-api.unit.js
@@ -169,7 +169,7 @@ describe('PluginAPI', () => {
 
   describe('_normalizeConfig', () => {
     it('should copy meteor object to app', () => {
-      const expected = { meteor: { path: '../' }, app: { type: 'meteor', path: '../', docker: { image: 'kadirahq/meteord', imagePort: 3000, stopAppDuringPrepareBundle: true } } };
+      const expected = { meteor: { path: '../' }, app: { type: 'meteor', path: '../', docker: { image: 'zodern/meteor:0.6.1-root', imagePort: 3000, stopAppDuringPrepareBundle: true } } };
       const config = { meteor: { path: '../' } };
       const result = api._normalizeConfig(config);
 

From 70f1cfed54d3da6eb857d5c850c891f51b7471d4 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Tue, 19 Oct 2021 19:25:47 -0500
Subject: [PATCH 23/36] Enable useBuiltKit by default

---
 docs/docs.md                | 2 +-
 src/plugins/meteor/index.js | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/docs.md b/docs/docs.md
index 435dccef..1c3a1099 100644
--- a/docs/docs.md
+++ b/docs/docs.md
@@ -139,7 +139,7 @@ module.exports = {
       // make deploys more reliable and easier to troubleshoot
       prepareBundle: true,
 
-      // (optional, default is false) Uses the new docker image builder
+      // (optional, default is true) Uses the new docker image builder
       // during Prepare bundle. When enabled,
       // Prepare Bundle is much faster
       useBuildKit: true,
diff --git a/src/plugins/meteor/index.js b/src/plugins/meteor/index.js
index 96507e10..977a5b40 100644
--- a/src/plugins/meteor/index.js
+++ b/src/plugins/meteor/index.js
@@ -28,7 +28,8 @@ export function prepareConfig(config, api) {
 
   config.app.docker = defaultsDeep(config.app.docker, {
     image: config.app.dockerImage || 'zodern/meteor:0.6.1-root',
-    stopAppDuringPrepareBundle: true
+    stopAppDuringPrepareBundle: true,
+    useBuildKit: true
   });
 
   delete config.app.dockerImage;

From f95d7ff3a7c740e2a29e6024dfe24ad4509aedac Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Tue, 19 Oct 2021 20:25:27 -0500
Subject: [PATCH 24/36] Update getSessionsForServers for server groups

---
 src/plugin-api.js | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/plugin-api.js b/src/plugin-api.js
index 16fc7912..d179c3ce 100644
--- a/src/plugin-api.js
+++ b/src/plugin-api.js
@@ -487,7 +487,20 @@ export default class PluginAPI {
       this._loadSessions();
     }
 
-    return servers.map(name => this.sessions[name]);
+    let result = [];
+
+    servers.forEach(name => {
+      let session = this.sessions[name];
+      if (Array.isArray(session)) {
+        session.forEach(memberName => {
+          result.push(this.sessions[memberName]);
+        });
+      } else {
+        result.push(session);
+      }
+    });
+
+    return result;
   }
 
   async getManagerSession() {

From e259a2871fcaeae86512e34fbc284402cccfc6d7 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Tue, 19 Oct 2021 20:26:34 -0500
Subject: [PATCH 25/36] Add "mup meteor shell" command

---
 src/plugins/meteor/command-handlers.js | 56 ++++++++++++++++++++++++++
 src/plugins/meteor/commands.js         |  9 +++++
 2 files changed, 65 insertions(+)

diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js
index 97c10ad4..a438a39a 100644
--- a/src/plugins/meteor/command-handlers.js
+++ b/src/plugins/meteor/command-handlers.js
@@ -18,6 +18,7 @@ import debug from 'debug';
 import nodemiral from '@zodern/nodemiral';
 import { rollback } from './rollback';
 import state from './state';
+import { Client } from 'ssh2';
 
 
 const log = debug('mup:module:meteor');
@@ -664,6 +665,61 @@ export async function debugApp(api) {
   });
 }
 
+export async function meteorShell(api) {
+  const { app } = api.getConfig();
+  const expandedServers = api.expandServers(app.servers);
+  let serverOption = api.getArgs()[1];
+
+  // Check how many sessions are enabled. Usually is all servers,
+  // but can be reduced by the `--servers` option
+  const enabledSessions = api.getSessionsForServers(Object.keys(app.servers))
+    .filter(session => session);
+
+  if (!(serverOption in expandedServers)) {
+    if (enabledSessions.length === 1) {
+      const selectedHost = enabledSessions[0]._host;
+      serverOption = Object.keys(expandedServers).find(key =>
+        expandedServers[key].server.host === selectedHost);
+    } else {
+      console.log('mup meteor shell <server>');
+      console.log('Available servers are:\n', Object.keys(expandedServers).join('\n '));
+      process.exitCode = 1;
+
+      return;
+    }
+  }
+
+  const server = expandedServers[serverOption].server;
+  const sshOptions = api._createSSHOptions(server);
+
+  const conn = new Client();
+  conn.on('ready', () => {
+    conn.exec(
+      `docker exec -it ${app.name} node ./meteor-shell.js`,
+      { pty: true },
+      (err, stream) => {
+        if (err) {
+          throw err;
+        }
+        stream.on('close', () => {
+          conn.end();
+          process.exit();
+        });
+
+        process.stdin.setRawMode(true);
+        process.stdin.pipe(stream);
+
+        stream.pipe(process.stdout);
+        stream.stderr.pipe(process.stderr);
+        stream.setWindow(process.stdout.rows, process.stdout.columns);
+
+        process.stdout.on('resize', () => {
+          stream.setWindow(process.stdout.rows, process.stdout.columns);
+        });
+      });
+  }).connect(sshOptions);
+}
+
 export async function destroy(api) {
   const config = api.getConfig();
   const options = api.getOptions();
diff --git a/src/plugins/meteor/commands.js b/src/plugins/meteor/commands.js
index db0c64b3..966d47d4 100644
--- a/src/plugins/meteor/commands.js
+++ b/src/plugins/meteor/commands.js
@@ -87,6 +87,15 @@ export const debug = {
   handler: commandHandlers.debugApp
 };
 
+export const shell = {
+  name: 'shell [server]',
+  description: 'Open production Meteor shell',
+  builder(yargs) {
+    yargs.strict(false);
+  },
+  handler: commandHandlers.meteorShell
+};
+
 export const versions = {
   description: 'List application versions',
   handler: commandHandlers.listVersions

From 9d0ca0fa1ade8f7326cbe10f3ec9edc8bad9e84a Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 20 Oct 2021 15:22:57 -0500
Subject: [PATCH 26/36] Add basic graceful shutdown for "mup restart"

---
 src/nodemiral.js                         | 16 +++++
 src/plugin-api.js                        | 19 +++++-
 src/plugins/meteor/assets/meteor-stop.sh |  1 +
 src/plugins/meteor/command-handlers.js   |  6 ++
 src/plugins/proxy/assets/upstream.sh     |  2 +-
 src/plugins/proxy/graceful-shutdown.js   | 74 ++++++++++++++++++++++++
 src/plugins/proxy/index.js               |  6 +-
 src/plugins/proxy/utils.js               |  8 ++-
 src/utils.js                             | 16 ++++-
 9 files changed, 137 insertions(+), 11 deletions(-)
 create mode 100644 src/plugins/proxy/graceful-shutdown.js

diff --git a/src/nodemiral.js b/src/nodemiral.js
index 661aba8b..1f795f90 100644
--- a/src/nodemiral.js
+++ b/src/nodemiral.js
@@ -87,8 +87,24 @@ function createCallback(cb, varsMapper) {
   };
 }
 
+// TODO: running hooks should be an option for executeScript instead of
+// a separate task
+async function runDuringHooks(session, options, callback) {
+  console.dir(options);
+  const pluginApi = options._getMupApi();
+  // console.dir(pluginApi);
+  try {
+    await pluginApi._runDuringHooks(options.hookName, session);
+  } catch (e) {
+    return callback(e);
+  }
+
+  callback();
+}
+
 nodemiral.registerTask('copy', copy);
 nodemiral.registerTask('executeScript', executeScript);
+nodemiral.registerTask('_runHook', runDuringHooks);
 
 const oldApplyTemplate = nodemiral.session.prototype._applyTemplate;
 // Adds support for using include with ejs
diff --git a/src/plugin-api.js b/src/plugin-api.js
index d179c3ce..05b2b2a9 100644
--- a/src/plugin-api.js
+++ b/src/plugin-api.js
@@ -98,6 +98,8 @@ export default class PluginAPI {
       opts.showDuration = this.profileTasks;
     }
 
+    opts._mupPluginApi = this;
+
     return utils.runTaskList(list, sessions, opts);
   }
 
@@ -247,7 +249,7 @@ export default class PluginAPI {
       process.exit(1);
     }
   }
-  _runHooks = async function(handlers, hookName) {
+  _runHooks = async function(handlers, hookName, secondArg) {
     const messagePrefix = `> Running hook ${hookName}`;
 
     for (const hookHandler of handlers) {
@@ -257,7 +259,7 @@ export default class PluginAPI {
       }
       if (typeof hookHandler.method === 'function') {
         try {
-          await hookHandler.method(this, nodemiral);
+          await hookHandler.method(this, secondArg || nodemiral);
         } catch (e) {
           this._commandErrorHandler(e);
         }
@@ -273,6 +275,19 @@ export default class PluginAPI {
       }
     }
   }
+  async _runDuringHooks(name, session) {
+    const hookName = `during.${name}`;
+
+    if (this.program['show-hook-names']) {
+      console.log(chalk.yellow(`Hook: ${hookName}`));
+    }
+
+    if (hookName in hooks) {
+      const hookList = hooks[hookName];
+
+      await this._runHooks(hookList, name, { session });
+    }
+  }
   _runPreHooks = async function(name) {
     const hookName = `pre.${name}`;
 
diff --git a/src/plugins/meteor/assets/meteor-stop.sh b/src/plugins/meteor/assets/meteor-stop.sh
index 9c7bea4c..c2e48448 100644
--- a/src/plugins/meteor/assets/meteor-stop.sh
+++ b/src/plugins/meteor/assets/meteor-stop.sh
@@ -2,6 +2,7 @@
 
 APPNAME=<%= appName %>
 
+sudo docker stop -t 30 $APPNAME || :
 sudo docker rm -f $APPNAME || :
 sudo docker rm -f $APPNAME-frontend || :
 sudo docker rm -f $APPNAME-nginx-letsencrypt || :
diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js
index a438a39a..5351b4df 100644
--- a/src/plugins/meteor/command-handlers.js
+++ b/src/plugins/meteor/command-handlers.js
@@ -546,6 +546,9 @@ export async function restart(api) {
   if (api.swarmEnabled()) {
     api.tasks.addRestartService(list, { name: appConfig.name });
   } else {
+    list._runHook('Stopping app', {
+      hookName: 'app.shutdown'
+    });
     list.executeScript('Stop Meteor', {
       script: api.resolvePath(__dirname, 'assets/meteor-stop.sh'),
       vars: {
@@ -554,6 +557,9 @@ export async function restart(api) {
     });
     addStartAppTask(list, api);
     checkAppStarted(list, api);
+    list._runHook('Finish starting', {
+      hookName: 'app.start-instance'
+    });
   }
 
 
diff --git a/src/plugins/proxy/assets/upstream.sh b/src/plugins/proxy/assets/upstream.sh
index a7b8b3d0..d620d1fc 100644
--- a/src/plugins/proxy/assets/upstream.sh
+++ b/src/plugins/proxy/assets/upstream.sh
@@ -23,7 +23,7 @@ cat <<"EOT" > /opt/$PROXYNAME/upstream/$APPNAME
 ip_hash;
 <% } %>
 <% for(var index in hostnames) { %>
-server <%= hostnames[index] %>:<%= port %>;
+server <%= hostnames[index].host %>:<%= port %> <%= hostnames[index].params %>;
 <% } %>
 EOT
 
diff --git a/src/plugins/proxy/graceful-shutdown.js b/src/plugins/proxy/graceful-shutdown.js
new file mode 100644
index 00000000..7b9b8745
--- /dev/null
+++ b/src/plugins/proxy/graceful-shutdown.js
@@ -0,0 +1,74 @@
+import { PROXY_CONTAINER_NAME } from './command-handlers';
+import { getLoadBalancingHosts, getSessions } from './utils';
+
+function runScript(sessions, script, vars) {
+  const promises = sessions.map(session =>
+    new Promise((resolve, reject) => {
+      session.executeScript(script, {
+        vars
+      }, (err, code, output) => {
+        console.log(err, code, output);
+        if (code > 0) {
+          return reject(output);
+        }
+
+        resolve();
+      });
+    }));
+
+  return Promise.all(promises);
+}
+
+function updateUpstreams(api, toDrain) {
+  const {
+    app: appConfig,
+    proxy: config
+  } = api.getConfig();
+  const hostnames = getLoadBalancingHosts(
+    api.expandServers(appConfig.servers),
+    toDrain ? [toDrain] : undefined
+  );
+  const domains = config.domains.split(',');
+
+  const proxySessions = getSessions(api);
+
+  // TODO: we don't need to update the domains
+  return runScript(
+    proxySessions,
+    api.resolvePath(__dirname, 'assets/upstream.sh'),
+    {
+      domains,
+      name: appConfig.name,
+      setUpstream: !api.swarmEnabled() && config.loadBalancing,
+      stickySessions: config.stickySessions !== false,
+      proxyName: PROXY_CONTAINER_NAME,
+      port: appConfig.env.PORT,
+      hostnames
+    }
+  );
+}
+
+export async function gracefulShutdown(api, { session }) {
+  console.log('graceful shutdown');
+  const {
+    proxy
+  } = api.getConfig();
+
+  if (!proxy || !proxy.loadBalancing) {
+    return;
+  }
+
+  await updateUpstreams(api, session._host);
+}
+
+export async function readdInstance(api) {
+  const {
+    proxy
+  } = api.getConfig();
+
+  if (!proxy || !proxy.loadBalancing) {
+    return;
+  }
+
+  await updateUpstreams(api);
+}
diff --git a/src/plugins/proxy/index.js b/src/plugins/proxy/index.js
index b7617432..4e1f4bac 100644
--- a/src/plugins/proxy/index.js
+++ b/src/plugins/proxy/index.js
@@ -2,6 +2,7 @@ import * as _commands from './commands';
 import { addProxyEnv, getLoadBalancingHosts, getSessions, normalizeUrl } from './utils';
 import { PROXY_CONTAINER_NAME, updateProxyForLoadBalancing } from './command-handlers';
 import validator from './validate';
+import { gracefulShutdown, readdInstance } from './graceful-shutdown';
 
 export const description = 'Setup and manage reverse proxy and ssl';
 
@@ -125,7 +126,9 @@ export const hooks = {
     }
   },
   'post.reconfig': configureServiceHook,
-  'post.proxy.setup': configureServiceHook
+  'post.proxy.setup': configureServiceHook,
+  'during.app.shutdown': gracefulShutdown,
+  'during.app.start-instance': readdInstance
 };
 
 export function swarmOptions(config) {
@@ -169,7 +172,6 @@ export async function checkSetup(api) {
   let upstream = [];
 
   if (config.loadBalancing) {
-    console.dir(config.app);
     upstream = getLoadBalancingHosts(
       api.expandServers(config.app.servers)
     );
diff --git a/src/plugins/proxy/utils.js b/src/plugins/proxy/utils.js
index fe582520..6fde4d71 100644
--- a/src/plugins/proxy/utils.js
+++ b/src/plugins/proxy/utils.js
@@ -44,8 +44,10 @@ export function normalizeUrl(config, env) {
   return _config.app.env.ROOT_URL;
 }
 
-export function getLoadBalancingHosts(expandedServers) {
-  return Object.values(expandedServers).map(({ server }) =>
-    server.privateIp || server.host
+export function getLoadBalancingHosts(expandedServers, drainingHosts = []) {
+  return Object.values(expandedServers).map(({ server }) => ({
+    host: server.privateIp || server.host,
+    params: drainingHosts.includes(server.host) ? 'down' : ''
+  })
   );
 }
diff --git a/src/utils.js b/src/utils.js
index 8091b2cd..7e5b5d97 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -33,11 +33,21 @@ export function runTaskList(list, sessions, opts) {
     delete opts.verbose;
   }
 
-  if (opts && opts.showDuration) {
+  if (opts) {
+    let pluginApi = opts._mupPluginApi;
     list._taskQueue.forEach(task => {
       task.options = task.options || {};
-      task.options.showDuration = true;
+
+      if (task.type === '_runHook') {
+        task.options._getMupApi = () => () => pluginApi;
+      }
+
+      if (opts.showDuration) {
+        task.options.showDuration = true;
+      }
     });
+
+    delete opts._mupPluginApi;
     delete opts.showDuration;
   }
 
@@ -240,7 +250,7 @@ export function forwardPort({
   remotePort,
   onReady,
   onError,
-  onConnection = () => {}
+  onConnection = () => { }
 }) {
   const sshOptions = createSSHOptions(server);
   const netServer = net.createServer(netConnection => {

From b3534ba2ed923dbd499b3bbee563107195559d9f Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Tue, 11 Jan 2022 15:20:59 -0600
Subject: [PATCH 27/36] Add basic support for resizing servers

---
 .../default/server-groups/digital-ocean.js    | 114 +++++++++++++++---
 src/plugins/default/server-groups/index.js    |  22 ++--
 2 files changed, 112 insertions(+), 24 deletions(-)

diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
index 0b0db7a2..e80accbd 100644
--- a/src/plugins/default/server-groups/digital-ocean.js
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -44,23 +44,36 @@ export default class DigitalOcean {
 
     const good = [];
     const wrong = [];
+    const toResize = [];
 
     servers.forEach(server => {
       const droplet = server.__droplet;
 
+      if (droplet.region.slug !== this.config.region) {
+        wrong.push(server);
+
+        return;
+      }
+
       if (
-        droplet.size_slug !== this.config.size ||
-        droplet.region.slug !== this.config.region
+        droplet.size_slug !== this.config.size
       ) {
-        wrong.push(server);
-      } else {
-        good.push(server);
+        if (this.config._resize) {
+          toResize.push(server);
+        } else {
+          wrong.push(server);
+        }
+
+        return;
       }
+
+      good.push(server);
     });
 
     return {
       wrong,
-      good
+      good,
+      toResize
     };
   }
 
@@ -77,6 +90,11 @@ export default class DigitalOcean {
     let fingerprint = await this._setupPublicKey();
     const names = [];
 
+    let size = this.config.size;
+    if (this.config._resize) {
+      size = this.config._resize.initialSize;
+    }
+
     while (names.length < count) {
       names.push(generateName(this.name));
     }
@@ -84,7 +102,7 @@ export default class DigitalOcean {
     const data = {
       names,
       region: this.config.region,
-      size: this.config.size,
+      size,
 
       // TODO: pick image from API
       image: 'ubuntu-20-04-x64',
@@ -106,7 +124,11 @@ export default class DigitalOcean {
     );
 
     const ids = result.data.droplets.map(droplet => droplet.id);
-    await Promise.all(ids.map(id => this._waitForDropletActive(id)));
+    await Promise.all(ids.map(id => this._waitForStatus(id, 'active')));
+
+    if (size !== this.config.size) {
+      await Promise.all(ids.map(id => this.resizeServer(id, this.config.size)));
+    }
 
     return this.getServers(ids);
   }
@@ -150,25 +172,89 @@ export default class DigitalOcean {
     return fingerprint;
   }
 
-  async _waitForDropletActive(id) {
-    const TEN_MINUTES = 1000 * 60 * 10;
-    const timeoutAt = Date.now() + TEN_MINUTES;
+  // Default timeout is 10 minutes
+  async _waitForStatus(dropletId, desiredStatus, timeout = 1000 * 60 * 10) {
+    const timeoutAt = Date.now() + timeout;
 
     while (Date.now() < timeoutAt) {
       const response = await this._request(
         'get',
-        `droplets/${id}`
+        `droplets/${dropletId}`
       );
       const status = response.data.droplet.status;
 
-      if (status === 'active') {
+      if (status === desiredStatus) {
         return;
       }
 
       await new Promise(resolve => setTimeout(resolve, 1000 * 10));
     }
 
-    throw new Error(`Timed out waiting for droplet ${id} to become active`);
+    throw new Error(`Timed out waiting for droplet ${dropletId} to become active`);
+  }
+
+  async resizeServer(dropletId) {
+    await this._shutdownDroplet(dropletId);
+
+    const result = await this._request(
+      'post',
+      `droplets/${dropletId}/actions`,
+      {
+        type: 'resize',
+        disk: false,
+        size: this.config.size
+      }
+    );
+
+    const actionId = result.data.action.id;
+
+    let timeoutAt = Date.now() + (1000 * 60 * 10);
+    while (Date.now() < timeoutAt) {
+      const { data } = await this._request(
+        'get',
+        `droplets/${dropletId}/actions/${actionId}`
+      );
+
+      if (data.action.status === 'completed') {
+        break;
+      }
+
+      await new Promise(resolve => setTimeout(resolve, 1000 * 10));
+    }
+
+    await this._request(
+      'post',
+      `droplets/${dropletId}/actions`,
+      {
+        type: 'power_on'
+      }
+    );
+
+    await this._waitForStatus(dropletId, 'active');
+  }
+
+  async _shutdownDroplet(dropletId) {
+    await this._request(
+      'post',
+      `droplets/${dropletId}/actions`,
+      {
+        type: 'shutdown'
+      }
+    );
+
+    try {
+      await this._waitForStatus(dropletId, 'off', 1000 * 60 * 3);
+    } catch (e) {
+      console.log(e);
+      await this._request(
+        'post',
+        `droplets/${dropletId}/actions`,
+        {
+          type: 'power_off'
+        }
+      );
+      await this._waitForStatus(dropletId, 'off');
+    }
   }
 
   _request(method, path, data) {
diff --git a/src/plugins/default/server-groups/index.js b/src/plugins/default/server-groups/index.js
index 892f0537..9a4d2410 100644
--- a/src/plugins/default/server-groups/index.js
+++ b/src/plugins/default/server-groups/index.js
@@ -10,24 +10,19 @@ function createSourceConfig(SourceAPI) {
     },
     async upToDate({ name, groupConfig, list }, pluginApi) {
       const api = new SourceAPI(name, groupConfig, pluginApi);
-      const { wrong, good } = await api.compareServers(list);
+      const { wrong, good, toResize = [] } = await api.compareServers(list);
 
-      return wrong.length === 0 && good.length === groupConfig.count;
+      return wrong.length === 0 &&
+        good.length === groupConfig.count &&
+        toResize.length === 0;
     },
     async update({ name, groupConfig }, pluginApi) {
       const api = new SourceAPI(name, groupConfig, pluginApi);
-      const { wrong, good } = await api.compareServers();
+      const { wrong, good, toResize = [] } = await api.compareServers();
 
       const addCount = Math.max(0, groupConfig.count - good.length);
       const goodRemoveCount = Math.max(0, good.length - groupConfig.count);
 
-      // TODO: finish implementing this
-      // If the region or type changed, all of the servers are wrong.
-      // We want to temporarily keep some of the wrong servers so they can
-      // handle requests until the new servers are ready
-      // const min = Math.ceil(groupConfig.count / 2);
-      // const tempCount = Math.min(wrong.length, min - groupConfig.count);
-
       if (goodRemoveCount > 0) {
         console.log(`=> Removing ${goodRemoveCount} servers for ${name}`);
         await api.removeServers(good.slice(0, goodRemoveCount));
@@ -45,6 +40,13 @@ function createSourceConfig(SourceAPI) {
         await waitForServers(created, pluginApi);
         console.log(`=> Finished creating ${addCount} servers for ${name}`);
       }
+
+      for (const server of toResize) {
+        console.log(`=> Resizing ${server.__droplet.name} for ${name}`);
+        await api.resizeServer(server.__droplet.id);
+        await waitForServers([server], pluginApi);
+        console.log(`=> Finished resizing ${server.__droplet.name} for ${name}`);
+      }
     }
   };
 }

From 6b2ce752b6b68406de272cb7a149a0a69cf88dd8 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Fri, 21 Jan 2022 11:25:03 -0600
Subject: [PATCH 28/36] Replace __tagPrefix with __tag option

---
 src/plugins/default/server-groups/digital-ocean.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
index e80accbd..cc532774 100644
--- a/src/plugins/default/server-groups/digital-ocean.js
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -9,8 +9,7 @@ export default class DigitalOcean {
 
     this.publicKeyPath = pluginApi.resolvePath(this.config.sshKey.public);
 
-    const tagPrefix = groupConfig.__tagPrefix || 'mup-';
-    this.tag = `${tagPrefix}-${this.name}`;
+    this.tag = groupConfig.__tag || `mup-${this.name}`;
   }
 
   async getServers(ids) {

From 2dcb060254cc934bc553992e76abbff5c9cc0b11 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Fri, 21 Jan 2022 11:26:39 -0600
Subject: [PATCH 29/36] Release 1.6.0-beta.2

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 4ac9e988..d34e3f56 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mup",
-  "version": "1.6.0-beta.1",
+  "version": "1.6.0-beta.2",
   "description": "Production Quality Meteor Deployments",
   "main": "lib/index.js",
   "repository": {

From d85a458687d522fda482471a9a4cf354669c25b1 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 20 Apr 2022 14:33:31 -0500
Subject: [PATCH 30/36] Remove unnecessary log

---
 src/check-setup.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/check-setup.js b/src/check-setup.js
index 1edcbd58..c5896cd8 100644
--- a/src/check-setup.js
+++ b/src/check-setup.js
@@ -48,7 +48,6 @@ export async function checkSetup(pluginApi) {
   const promises = [];
 
   bySession.forEach((config, session) => {
-    console.log(session._host, config);
     const promise = new Promise(resolve => {
       session.executeScript(
         pluginApi.resolvePath(__dirname, './tasks/assets/check-setup.sh'),

From ddb9b7268ddad4f955cdc1c6f01ff201df859ec3 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Wed, 20 Apr 2022 14:45:03 -0500
Subject: [PATCH 31/36] Release 1.6.0-beta.3

---
 package-lock.json | 4 ++--
 package.json      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index eb65caba..28d428f3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "mup",
-  "version": "1.5.8-beta.1",
+  "version": "1.6.0-beta.1",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mup",
-      "version": "1.5.8-beta.1",
+      "version": "1.6.0-beta.1",
       "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
diff --git a/package.json b/package.json
index 6a03a5fe..4dd115e3 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mup",
-  "version": "1.6.0-beta.2",
+  "version": "1.6.0-beta.3",
   "description": "Production Quality Meteor Deployments",
   "main": "lib/index.js",
   "repository": {

From 29b7872eecc7a20294da2d215fd7e13ad60786f3 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Mon, 27 Jun 2022 16:22:42 -0500
Subject: [PATCH 32/36] v1.6.0-beta.4

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 645744c4..4318d7ef 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mup",
-  "version": "1.6.0-beta.3",
+  "version": "1.6.0-beta.4",
   "description": "Production Quality Meteor Deployments",
   "main": "lib/index.js",
   "repository": {

From 022aaefdc3d61ab1d535a541b54db555723b2011 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Tue, 28 Jun 2022 10:45:55 -0500
Subject: [PATCH 33/36] Update ssh2 imports

---
 src/plugins/default/server-groups/utils.js | 2 +-
 src/plugins/meteor/command-handlers.js     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plugins/default/server-groups/utils.js b/src/plugins/default/server-groups/utils.js
index 6f4df62d..329b836e 100644
--- a/src/plugins/default/server-groups/utils.js
+++ b/src/plugins/default/server-groups/utils.js
@@ -1,5 +1,5 @@
 import crypto from 'crypto';
-import { Client } from 'ssh2';
+import { Client } from 'ssh2-classic';
 
 export function generateName(groupName) {
   const randomString = crypto.randomBytes(4).toString('hex');
diff --git a/src/plugins/meteor/command-handlers.js b/src/plugins/meteor/command-handlers.js
index 5351b4df..c417880b 100644
--- a/src/plugins/meteor/command-handlers.js
+++ b/src/plugins/meteor/command-handlers.js
@@ -18,7 +18,7 @@ import debug from 'debug';
 import nodemiral from '@zodern/nodemiral';
 import { rollback } from './rollback';
 import state from './state';
-import { Client } from 'ssh2';
+import { Client } from 'ssh2-classic';
 
 
 const log = debug('mup:module:meteor');

From 66a60fe0830945571e86fcac485465c65568405f Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Tue, 28 Jun 2022 10:46:10 -0500
Subject: [PATCH 34/36] v1.6.0-beta.5

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 4318d7ef..6edd9f49 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "mup",
-  "version": "1.6.0-beta.4",
+  "version": "1.6.0-beta.5",
   "description": "Production Quality Meteor Deployments",
   "main": "lib/index.js",
   "repository": {

From fba96bacb0bce3ac4f91cdb4271cf4a04f14c784 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Fri, 4 Oct 2024 09:53:51 -0500
Subject: [PATCH 35/36] Redact Authorization header

---
 src/plugins/default/server-groups/digital-ocean.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/plugins/default/server-groups/digital-ocean.js b/src/plugins/default/server-groups/digital-ocean.js
index cc532774..93c8d83d 100644
--- a/src/plugins/default/server-groups/digital-ocean.js
+++ b/src/plugins/default/server-groups/digital-ocean.js
@@ -264,6 +264,12 @@ export default class DigitalOcean {
       headers: {
         Authorization: `Bearer ${this.config.token}`
       }
+    }).catch(err => {
+      if (err.config && err.config.headers) {
+        err.config.headers.Authorization = '<redacted>';
+      }
+
+      throw err;
     });
   }
 }

From 1e18967f9b84a2cd9321c63da4b9cca5bed13403 Mon Sep 17 00:00:00 2001
From: zodern <zodern@users.noreply.github.com>
Date: Fri, 4 Oct 2024 10:04:49 -0500
Subject: [PATCH 36/36] Log response time in nginx

---
 src/plugins/proxy/assets/nginx.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plugins/proxy/assets/nginx.tmpl b/src/plugins/proxy/assets/nginx.tmpl
index ba1a485b..6d9db078 100644
--- a/src/plugins/proxy/assets/nginx.tmpl
+++ b/src/plugins/proxy/assets/nginx.tmpl
@@ -62,7 +62,7 @@ gzip_types text/plain text/css application/javascript application/json applicati
 
 log_format vhost '$host $remote_addr - $remote_user [$time_local] '
                  '"$request" $status $body_bytes_sent '
-                 '"$http_referer" "$http_user_agent"';
+                 '"$http_referer" "$http_user_agent" $request_time $upstream_response_time $pipe';
 
 access_log off;
 
@@ -362,4 +362,4 @@ server {
 {{ end }}
 
 {{ end }}
-{{ end }}
\ No newline at end of file
+{{ end }}