use unix socket instead of TCP socket (#29)

* use unix socket instead of TCP socket

* docs

* switch back to default context on stop and delete

* mock getuser

* better exception handling

README.md

@@ -75,12 +75,12 @@ How it works: two processes are run, a sync and a tunnel process.
 
 ## Daily Running
 
-Note: QUIT Docker Desktop (or any local docker-agent equivalent) when using the remote agent
-
 1. Start the remote-docker ec2 instance
     ```bash
     remote-docker-aws start
     ```
+   This will automatically switch the docker context for you. If you want to switch
+   back to the default agent run `docker context use default`
 
 1. In one terminal start the tunnel so that the ports you need to connect to are exposed
     ```bash
@@ -106,21 +106,7 @@ Note: QUIT Docker Desktop (or any local docker-agent equivalent) when using the
     remote-docker-aws sync
     ```
 
-1. Make sure to set `DOCKER_HOST`:
-    ```bash
-    # In the terminal you use docker, or add to ~/.bashrc so it applies automatically
-    export DOCKER_HOST="tcp://localhost:23755"
-    ```
-
-   Now you can use docker as you would normally:
-   - `docker build -t myapp .`
-   - `docker-compose up`
-   - etc.
-
-   You can usually skip starting your services again since when the instance
-   boots, it will start up docker and resume where it left off from the day before.
-
-1. Develop and code! All services should be accessible and usable as usual
+1. Develop and code! All services should be accessible and usable as usual (eg: `docker ps`, `docker-compose up`, etc.)
 as long as you are running `remote-docker-aws tunnel` and are forwarding the ports you need
 
 1. When you're done for the day don't forget to stop the instance to save money:
@@ -227,4 +213,8 @@ Nothing else used should incur any cost with reasonable usage
 
 ## Notes
 - See `remote-docker-aws --help` for more information on the commands available
+- The unison version running on the server and running locally have to
+match. If one of them updates to a newer version, you should update the other.
+This can be done locally via `brew upgrade unison`, and to update the remote unison version:
+`rd ssh` then `brew upgrade unison`
 

src/remote_docker_aws/__init__.py

@@ -1,8 +1,17 @@
+import sys
+
 from sceptre.cli.helpers import setup_logging
 
 from .cli_commands import cli
+from .exceptions import RemoteDockerException
+from .util import logger
 
 
 def main():
     setup_logging(debug=False, no_colour=False)
-    cli()
+
+    try:
+        cli()
+    except RemoteDockerException as e:
+        logger.error(e)
+        sys.exit(-1)

src/remote_docker_aws/cli_commands.py

@@ -62,13 +62,15 @@ def cmd_ssh(client: RemoteDockerClient, ssh_options=None, ssh_cmd=None):
 def cmd_start(client: RemoteDockerClient):
     """Start the remote agent instance"""
     print(client.start_instance())
+    client.use_remote_context()
 
 
 @cli.command(name="stop")
 @pass_config
 def cmd_stop(client: RemoteDockerClient):
     """Stop the remote agent instance"""
     print(client.stop_instance())
+    client.use_default_context()
 
 
 @cli.command(name="ip")
@@ -90,6 +92,7 @@ def cmd_create_keypair(client: RemoteDockerClient):
 def cmd_create(client: RemoteDockerClient):
     """Provision a new ec2 instance to use as the remote agent"""
     print(client.create_instance())
+    client.use_remote_context()
 
 
 @cli.command(name="delete")
@@ -104,6 +107,7 @@ def cmd_delete(client: RemoteDockerClient):
 
     click.confirm("Are you sure you want to delete your instance?", abort=True)
     print(client.delete_instance())
+    client.use_default_context()
 
 
 @cli.command(
@@ -162,3 +166,14 @@ def enable_termination_protection(client: RemoteDockerClient):
     the API and AWS console GUI"
     """
     client.enable_termination_protection()
+
+
+@cli.command(
+    name="context",
+)
+@pass_config
+def use_remote_context(client: RemoteDockerClient):
+    """
+    Creates and switches to the remote-docker context
+    """
+    client.use_remote_context()

src/remote_docker_aws/constants.py

@@ -3,8 +3,6 @@
 from typing import Dict
 
 
-DOCKER_PORT_FORWARD = {"23755": "2375"}
-
 KEY_PAIR_NAME = "remote-docker-keypair"
 INSTANCE_USERNAME = "ubuntu"
 # Used to identify the ec2 instance

src/remote_docker_aws/core.py

@@ -1,13 +1,13 @@
 import os
 import shlex
 import subprocess
+from getpass import getuser
 from typing import Dict, List
 
 from unison_gitignore.parser import GitIgnoreToUnisonIgnore
 
 from .config import RemoteDockerConfigProfile
 from .constants import (
-    DOCKER_PORT_FORWARD,
     INSTANCE_USERNAME,
     PORT_MAP_TYPE,
 )
@@ -96,8 +96,13 @@ def start_tunnel(
             f" -i {self.ssh_key_path} {self.instance.username}@{ip}"
         )
 
-        for port_from, port_to in DOCKER_PORT_FORWARD.items():
-            cmd_s += f" -L localhost:{port_from}:localhost:{port_to}"
+        target_sock = "/var/run/remote-docker.sock"
+        cmd_s += (
+            f" -L {target_sock}:/var/run/docker.sock"
+            " -o StreamLocalBindUnlink=yes"
+            " -o PermitLocalCommand=yes"
+            f" -o LocalCommand='sudo chown {getuser()} {target_sock}'"
+        )
 
         for _name, port_mappings in local_forwards.items():
             for port_from, port_to in port_mappings.items():
@@ -140,6 +145,29 @@ def ssh_run(self, *, ssh_cmd: str):
     def create_keypair(self) -> Dict:
         return self.instance.create_keypair(self.ssh_key_path)
 
+    def use_remote_context(self):
+        logger.info("Switching docker context to remote-docker")
+
+        subprocess.run(
+            (
+                "docker context inspect remote-docker &>/dev/null || "
+                "docker context create"
+                " --docker host=unix:///var/run/remote-docker.sock remote-docker"
+            ),
+            check=True,
+            shell=True,
+        )
+        subprocess.run(
+            "docker context use remote-docker >/dev/null",
+            check=True,
+            shell=True,
+        )
+
+    def use_default_context(self):
+        logger.info("Switching docker context to default")
+
+        subprocess.run("docker context use default >/dev/null", check=True, shell=True)
+
     def _get_unison_cmd(
         self,
         *,

src/remote_docker_aws/exceptions.py

@@ -0,0 +1,6 @@
+class RemoteDockerException(RuntimeError):
+    pass
+
+
+class InstanceNotRunning(RemoteDockerException):
+    pass

src/remote_docker_aws/providers.py

@@ -14,6 +14,7 @@
     AWS_REGION_TO_UBUNTU_AMI_MAPPING,
     SCEPTRE_PATH,
 )
+from .exceptions import InstanceNotRunning, RemoteDockerException
 from .util import logger, wait_until_port_is_open
 
 
@@ -83,6 +84,7 @@ def _build_ssh_cmd(self, ssh_key_path: str, ssh_cmd=None, options=None):
             f" {options} {self.username}@{self.get_ip()} {ssh_cmd}"
         )
 
+        logger.debug("Running: %s", cmd_s)
         return shlex.split(cmd_s)
 
 
@@ -124,11 +126,11 @@ def _get_instance(self) -> Dict:
         ]
 
         if len(valid_reservations) == 0:
-            raise RuntimeError(
+            raise RemoteDockerException(
                 "There are no valid reservations, did you create the instance?"
             )
         if len(valid_reservations) > 1:
-            raise RuntimeError(
+            raise RemoteDockerException(
                 "There is more than one reservation found that matched, not sure what to do"
             )
 
@@ -137,6 +139,10 @@ def _get_instance(self) -> Dict:
         return instances[0]
 
     def get_ip(self) -> str:
+        if not self.is_running():
+            raise InstanceNotRunning(
+                "Instance is not running. start it with `rd start`"
+            )
         return self._get_instance()["PublicIpAddress"]
 
     def get_instance_id(self) -> str:
@@ -246,8 +252,7 @@ def _bootstrap_instance(self, ssh_key_path: str):
         && sudo sysctl -w net.core.somaxconn=4096
         && sudo apt-get -y update
         && sudo apt-get -y install build-essential curl file git docker.io
-        && "sudo sed -i -e '/ExecStart=/ s/fd:\/\//127\.0\.0\.1:2375/' '/lib/systemd/system/docker.service'"
-        && sudo cp /lib/systemd/system/docker.service /etc/systemd/system/docker.service
+        && sudo usermod -aG docker ubuntu
         && sudo systemctl daemon-reload
         && sudo systemctl restart docker.service
         && sudo systemctl enable docker.service

tests/conftest.py

@@ -27,3 +27,26 @@ def ensure_aws_is_mocked():
 def ensure_sleep_is_mocked():
     with mock.patch("time.sleep"):
         yield
+
+
+@pytest.fixture(autouse=True)
+def mock_exec():
+    with mock.patch("os.execvp", autospec=True) as mock_exec_:
+        yield mock_exec_
+
+
+@pytest.fixture(autouse=True)
+def mock_run():
+    with mock.patch("subprocess.run", autospec=True) as mock_run_:
+        yield mock_run_
+
+
+@pytest.fixture
+def mock_user():
+    return "test_user"
+
+
+@pytest.fixture(autouse=True)
+def mock_getuser(mock_user):
+    with mock.patch("remote_docker_aws.core.getuser", return_value=mock_user):
+        yield

tests/test_cli.py

@@ -1,4 +1,3 @@
-import os
 import subprocess
 from contextlib import contextmanager
 from unittest import mock
@@ -14,10 +13,6 @@
 AWS_REGION = "ca-central-1"
 
 
-patch_exec = mock.patch("os.execvp", autospec=True)
-patch_run = mock.patch("subprocess.run", autospec=True)
-
-
 def test_cli_entrypoint_runs_successfully():
     output = subprocess.check_output("remote-docker-aws --help", shell=True)
     assert output
@@ -51,24 +46,29 @@ def cli_runner(self):
             yield runner
 
     @pytest.fixture
-    def create_instance(self, cli_runner):
+    def create_instance(self, cli_runner, mock_exec, mock_run):
         def create():
             with mock.patch(
                 "remote_docker_aws.providers.wait_until_port_is_open", autospec=True
             ):
-                if not isinstance(os.execvp, mock.MagicMock):
-                    with mock.patch("os.execvp"):
-                        result = cli_runner.invoke(cli, ["create"])
-                else:
-                    result = cli_runner.invoke(cli, ["create"])
+                result = cli_runner.invoke(cli, ["create"])
+
+                assert mock_run.call_count == 2
+                assert mock_exec.call_count == 1
+                mock_run.reset_mock()
+                mock_exec.reset_mock()
+
                 return result
 
         return create
 
     @pytest.fixture
-    def delete_instance(self, cli_runner):
+    def delete_instance(self, cli_runner, mock_run):
         def delete():
             result = cli_runner.invoke(cli, ["delete"], input="y")
+
+            mock_run.reset_mock()
+
             return result
 
         return delete
@@ -92,7 +92,6 @@ def test_create_and_delete(self, create_instance, delete_instance):
         result = delete_instance()
         assert result.exit_code == 0
 
-    @patch_exec
     def test_ssh(self, mock_exec, cli_runner, instance):
         with instance():
             result = cli_runner.invoke(cli, ["ssh"])
@@ -115,6 +114,11 @@ def test_ip(self, cli_runner, instance):
         assert result.exit_code == 0
         assert result.stdout
 
+    def test_context(self, cli_runner, mock_run):
+        result = cli_runner.invoke(cli, ["context"])
+        assert result.exit_code == 0
+        assert mock_run.call_count == 2
+
     @mock.patch(
         "remote_docker_aws.core.RemoteDockerClient.create_keypair", autospec=True
     )
@@ -125,8 +129,7 @@ def test_create_keypair(self, mock_create_keypair, cli_runner, instance):
         assert mock_create_keypair.call_count == 1
 
     @pytest.mark.parametrize("local,remote", [(None, None), ("80:80", "3300:3300")])
-    @patch_run
-    def test_tunnel(self, mock_run, local, remote, cli_runner, instance):
+    def test_tunnel(self, local, remote, mock_run, cli_runner, instance):
         args = ["tunnel"]
 
         if local:
@@ -136,22 +139,21 @@ def test_tunnel(self, mock_run, local, remote, cli_runner, instance):
 
         with instance():
             result = cli_runner.invoke(cli, args)
-        assert result.exit_code == 0
-        mock_run.assert_called_once()
+            assert result.exit_code == 0
+            mock_run.assert_called_once()
 
     @pytest.mark.parametrize("directories", [(["/data/mock_dir1", "/data/mock_dir2"])])
-    @patch_exec
-    @patch_run
-    def test_sync(self, mock_run, mock_exec, directories, cli_runner, instance):
+    def test_sync(self, directories, mock_run, mock_exec, cli_runner, instance):
         args = ["sync"]
 
         if directories:
             args.extend(directories)
 
         with instance():
             result = cli_runner.invoke(cli, args)
-        assert result.exit_code == 0
-        assert mock_run.call_count == 2
+            assert result.exit_code == 0
+            assert mock_run.call_count == 2
+
         mock_exec.assert_called_once()
 
     @mock.patch(