From c8401f8d0752a1945d190f01641f6f5c6e58a1d7 Mon Sep 17 00:00:00 2001 From: Matt Brictson Date: Thu, 20 Jun 2024 17:24:44 -0700 Subject: [PATCH] Replace Vagrant with Docker Compose for running functional tests (#539) * Replace Vagrant with Docker Compose * Reenable functional tests in CI * Stream docker compose output while building image * Fix RuboCop issue and Ruby 2 compat * Wait for Docker container to start, to avoid test flake --- .docker/Dockerfile | 6 ++ .docker/ubuntu_setup.sh | 22 ++++++ .github/workflows/ci.yml | 27 +++++++ .gitignore | 1 - .rubocop_todo.yml | 7 -- CONTRIBUTING.md | 4 +- RELEASING.md | 2 +- Rakefile | 4 -- Vagrantfile | 24 ------- docker-compose.yml | 8 +++ test/boxes.json | 17 ----- .../backends/netssh_transfer_tests.rb | 2 +- test/functional/backends/test_netssh.rb | 2 +- ...sh_server_comes_up_for_functional_tests.rb | 24 ------- test/helper.rb | 46 ++---------- test/support/docker_wrapper.rb | 71 +++++++++++++++++++ test/support/vagrant_wrapper.rb | 64 ----------------- test/unit/test_command_map.rb | 16 ++--- 18 files changed, 151 insertions(+), 196 deletions(-) create mode 100644 .docker/Dockerfile create mode 100755 .docker/ubuntu_setup.sh delete mode 100644 Vagrantfile create mode 100644 docker-compose.yml delete mode 100644 test/boxes.json delete mode 100644 test/functional/test_ssh_server_comes_up_for_functional_tests.rb create mode 100644 test/support/docker_wrapper.rb delete mode 100644 test/support/vagrant_wrapper.rb diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 00000000..12d6d7b7 --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:22.04 +WORKDIR /provision +COPY ./ubuntu_setup.sh ./ +RUN ./ubuntu_setup.sh +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/.docker/ubuntu_setup.sh b/.docker/ubuntu_setup.sh new file mode 100755 index 00000000..99b5e95b --- /dev/null +++ b/.docker/ubuntu_setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +export DEBIAN_FRONTEND=noninteractive +apt -y update + +# Create `deployer` user that can sudo without a password +apt-get -y install sudo +adduser --disabled-password deployer < /dev/null +echo "deployer:topsecret" | chpasswd +echo "deployer ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Install and configure sshd +apt-get -y install openssh-server +{ + echo "Port 22" + echo "PasswordAuthentication yes" + echo "ChallengeResponseAuthentication no" +} >> /etc/ssh/sshd_config +mkdir /var/run/sshd +chmod 0755 /var/run/sshd diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05f50c14..27f0ac8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,3 +69,30 @@ jobs: bundler-cache: true - name: Run rubocop run: bundle exec rake lint + + functional: + runs-on: ubuntu-latest + strategy: + matrix: + ruby: ["2.0", "ruby"] + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run functional tests + run: bundle exec rake test:functional + + functional-all: + runs-on: ubuntu-latest + needs: [functional] + if: always() + steps: + - name: All tests ok + if: ${{ !(contains(needs.*.result, 'failure')) }} + run: exit 0 + - name: Some tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 diff --git a/.gitignore b/.gitignore index 7b0b67f0..8207258a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,5 @@ bin/rake .bundle .yardoc -.vagrant* test/tmp Gemfile.lock diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 242f9414..a680666f 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -102,7 +102,6 @@ Layout/IndentHash: Exclude: - 'test/functional/backends/test_local.rb' - 'test/functional/backends/test_netssh.rb' - - 'test/support/vagrant_wrapper.rb' - 'test/unit/formatters/test_custom.rb' - 'test/unit/formatters/test_pretty.rb' - 'test/unit/test_mapping_interaction_handler.rb' @@ -445,12 +444,6 @@ Style/MethodName: Exclude: - 'test/unit/test_color.rb' -# Offense count: 1 -# Cop supports --auto-correct. -Style/MutableConstant: - Exclude: - - 'Vagrantfile' - # Offense count: 1 # Cop supports --auto-correct. # Configuration parameters: Strict. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54710b5c..8e35b7a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,8 +24,8 @@ using unsupported features. ## Tests -SSHKit has a unit test suite and a functional test suite. Some functional tests run against -[Vagrant](https://www.vagrantup.com/) VMs. If possible, you should make sure that the +SSHKit has a unit test suite and a functional test suite. Some functional tests run using +[Docker](https://docs.docker.com/get-docker/). If possible, you should make sure that the tests pass for each commit by running `rake` in the sshkit directory. This is in case we need to cherry pick commits or rebase. You should ensure the tests pass, (preferably on the minimum and maximum ruby version), before creating a PR. diff --git a/RELEASING.md b/RELEASING.md index 5f035192..348d879d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -9,7 +9,7 @@ ## How to release 1. Run `bundle install` to make sure that you have all the gems necessary for testing and releasing. -2. **Ensure the tests are passing by running `rake test`.** If functional tests fail, ensure you have [Vagrant](https://www.vagrantup.com) installed and have started it with `vagrant up`. +2. **Ensure the tests are passing by running `rake test`.** If functional tests fail, ensure you have [Docker installed](https://docs.docker.com/get-docker/) and running. 3. Determine which would be the correct next version number according to [semver](http://semver.org/). 4. Update the version in `./lib/sshkit/version.rb`. 5. Commit the `version.rb` change with a message like "Preparing vX.Y.Z" diff --git a/Rakefile b/Rakefile index 1fea0323..8f5558e9 100644 --- a/Rakefile +++ b/Rakefile @@ -21,10 +21,6 @@ namespace :test do end -Rake::Task["test:functional"].enhance do - warn "Remember there are still some VMs running, kill them with `vagrant halt` if you are finished using them." -end - desc 'Run RuboCop lint checks' RuboCop::RakeTask.new(:lint) do |task| task.options = ['--lint'] diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 13d20725..00000000 --- a/Vagrantfile +++ /dev/null @@ -1,24 +0,0 @@ -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = 'bento/ubuntu-22.10' - - config.vm.boot_timeout = 600 # seconds - config.ssh.insert_key = false - config.vm.provision "shell", inline: <<-SHELL - echo 'ClientAliveInterval 3' >> /etc/ssh/sshd_config - echo 'ClientAliveCountMax 3' >> /etc/ssh/sshd_config - echo 'MaxAuthTries 6' >> /etc/ssh/sshd_config - service ssh restart - SHELL - - json_config_path = File.join("test", "boxes.json") - list = File.open(json_config_path).read - list = JSON.parse(list) - - list.each do |vm| - config.vm.define vm["name"] do |web| - web.vm.network "forwarded_port", guest: 22, host: vm["port"] - end - end -end diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..4ce0f79f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +name: sshkit + +services: + ssh_server: + build: + context: .docker + ports: + - "2122:22" diff --git a/test/boxes.json b/test/boxes.json deleted file mode 100644 index 57793583..00000000 --- a/test/boxes.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "name": "one", - "port": 3001, - "user": "vagrant", - "password": "vagrant", - "hostname": "localhost" - }, - { - "name": "two", - "port": 3002 - }, - { - "name": "three", - "port": 3003 - } -] diff --git a/test/functional/backends/netssh_transfer_tests.rb b/test/functional/backends/netssh_transfer_tests.rb index 32562443..796d10e6 100644 --- a/test/functional/backends/netssh_transfer_tests.rb +++ b/test/functional/backends/netssh_transfer_tests.rb @@ -11,7 +11,7 @@ def setup end def a_host - VagrantWrapper.hosts['one'] + DockerWrapper.host end def test_upload_and_then_capture_file_contents diff --git a/test/functional/backends/test_netssh.rb b/test/functional/backends/test_netssh.rb index 3a2f03d6..ee7000ad 100644 --- a/test/functional/backends/test_netssh.rb +++ b/test/functional/backends/test_netssh.rb @@ -15,7 +15,7 @@ def setup end def a_host - VagrantWrapper.hosts['one'] + DockerWrapper.host end def test_simple_netssh diff --git a/test/functional/test_ssh_server_comes_up_for_functional_tests.rb b/test/functional/test_ssh_server_comes_up_for_functional_tests.rb deleted file mode 100644 index d1e42ab3..00000000 --- a/test/functional/test_ssh_server_comes_up_for_functional_tests.rb +++ /dev/null @@ -1,24 +0,0 @@ -require 'helper' - -module SSHKit - - class TestHost < FunctionalTest - - def host - @_host ||= Host.new('') - end - - def test_that_it_works - assert true - end - - def test_creating_a_user_gives_us_back_his_private_key_as_a_string - skip 'It is not safe to create an user for non vagrant envs' unless VagrantWrapper.running? - keys = create_user_with_key(:peter) - assert_equal [:one, :two, :three], keys.keys - assert keys.values.all? - end - - end - -end diff --git a/test/helper.rb b/test/helper.rb index f46bdfed..162fbdaf 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -28,51 +28,13 @@ def flush_connections end class FunctionalTest < Minitest::Test - def setup - unless VagrantWrapper.running? - warn "Vagrant VMs are not running. Please, start it manually with `vagrant up`" - end - end - - private - - def create_user_with_key(username, password = :secret) - username, password = username.to_s, password.to_s - - keys = VagrantWrapper.hosts.collect do |_name, host| - Net::SSH.start(host.hostname, host.user, port: host.port, password: host.password) do |ssh| - - # Remove the user, make it again, force-generate a key for him - # short keys save us a few microseconds - ssh.exec!("sudo userdel -rf #{username}; true") # The `rescue nil` of the shell world - ssh.exec!("sudo useradd -m #{username}") - ssh.exec!("sudo echo y | ssh-keygen -b 1024 -f #{username} -N ''") - ssh.exec!("sudo chown vagrant:vagrant #{username}*") - ssh.exec!("sudo echo #{username}:#{password} | chpasswd") + require_relative "support/docker_wrapper" + return if DockerWrapper.running? - # Make the .ssh directory, change the ownership and the - ssh.exec!("sudo mkdir -p /home/#{username}/.ssh") - ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh") - ssh.exec!("sudo chmod 700 /home/#{username}/.ssh") - - # Move the key to authorized keys and chown and chmod it - ssh.exec!("sudo cat #{username}.pub > /home/#{username}/.ssh/authorized_keys") - ssh.exec!("sudo chown #{username}:#{username} /home/#{username}/.ssh/authorized_keys") - ssh.exec!("sudo chmod 600 /home/#{username}/.ssh/authorized_keys") - - key = ssh.exec!("cat /home/vagrant/#{username}") - - # Clean Up Files - ssh.exec!("sudo rm #{username} #{username}.pub") - - key - end - end - - Hash[VagrantWrapper.hosts.collect { |n, _h| n.to_sym }.zip(keys)] + DockerWrapper.start + DockerWrapper.wait_for_ssh_server end - end # diff --git a/test/support/docker_wrapper.rb b/test/support/docker_wrapper.rb new file mode 100644 index 00000000..2165e699 --- /dev/null +++ b/test/support/docker_wrapper.rb @@ -0,0 +1,71 @@ +require "socket" + +Minitest.after_run do + DockerWrapper.stop if DockerWrapper.running? +end + +module DockerWrapper + SSH_SERVER_PORT = 2122 + + class << self + def host + SSHKit::Host.new( + user: "deployer", + hostname: "localhost", + port: SSH_SERVER_PORT, + password: "topsecret", + ssh_options: host_verify_options + ) + end + + def running? + out, status = run_compose_command("ps --status running", false) + status.success? && out.include?("ssh_server") + end + + def start + run_compose_command("up -d") + end + + def stop + run_compose_command("down") + end + + def wait_for_ssh_server(retries=3) + Socket.tcp("localhost", SSH_SERVER_PORT, connect_timeout: 1).close + sleep(1) + rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT + retries -= 1 + sleep(2) && retry if retries.positive? + raise + end + + private + + def run_compose_command(command, echo=true) + $stderr.puts "[docker compose] #{command}" if echo + Open3.popen2e("docker compose #{command}") do |stdin, outerr, wait_thread| + stdin.close + output = Thread.new { capture_stream(outerr, echo) } + [output.value, wait_thread.value] + end + end + + def capture_stream(stream, echo=true) + buffer = String.new + while (line = stream.gets) + buffer << line + $stderr.puts("[docker compose] #{line}") if echo + end + buffer + end + + def host_verify_options + if Net::SSH::Version::MAJOR >= 5 + { verify_host_key: :never } + else + { paranoid: false } + end + end + end +end diff --git a/test/support/vagrant_wrapper.rb b/test/support/vagrant_wrapper.rb deleted file mode 100644 index fbf3ae4d..00000000 --- a/test/support/vagrant_wrapper.rb +++ /dev/null @@ -1,64 +0,0 @@ -class VagrantWrapper - class << self - def hosts - @vm_hosts ||= begin - result = {} - - boxes = boxes_list - - unless running? - boxes.map! do |box| - box['user'] = ENV['USER'] - box['port'] = '22' - box - end - end - - boxes.each do |vm| - result[vm['name']] = vm_host(vm) - end - - result - end - end - - def running? - @running ||= begin - status = `#{vagrant_binary} status` - status.include?('running') - end - end - - def boxes_list - json_config_path = File.join('test', 'boxes.json') - boxes = File.open(json_config_path).read - JSON.parse(boxes) - end - - def vagrant_binary - 'vagrant' - end - - private - - def vm_host(vm) - host_options = { - user: vm['user'] || 'vagrant', - hostname: vm['hostname'] || 'localhost', - port: vm['port'] || '22', - password: vm['password'] || 'vagrant', - ssh_options: host_verify_options - } - - SSHKit::Host.new(host_options) - end - - def host_verify_options - if Net::SSH::Version::MAJOR >= 5 - { verify_host_key: :never } - else - { paranoid: false } - end - end - end -end diff --git a/test/unit/test_command_map.rb b/test/unit/test_command_map.rb index 06d37dfc..38d58e00 100644 --- a/test/unit/test_command_map.rb +++ b/test/unit/test_command_map.rb @@ -27,26 +27,26 @@ def test_setter_procs def test_prefix map = CommandMap.new - map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].push("/home/deployer/.rbenv/bin/rbenv exec") map.prefix[:rake].push("bundle exec") - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_prefix_procs map = CommandMap.new - map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].push("/home/deployer/.rbenv/bin/rbenv exec") map.prefix[:rake].push(proc{ "bundle exec" }) - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_prefix_unshift map = CommandMap.new map.prefix[:rake].push("bundle exec") - map.prefix[:rake].unshift("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].unshift("/home/deployer/.rbenv/bin/rbenv exec") - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_indifferent_setter @@ -59,10 +59,10 @@ def test_indifferent_setter def test_indifferent_prefix map = CommandMap.new - map.prefix[:rake].push("/home/vagrant/.rbenv/bin/rbenv exec") + map.prefix[:rake].push("/home/deployer/.rbenv/bin/rbenv exec") map.prefix["rake"].push("bundle exec") - assert_equal map[:rake], "/home/vagrant/.rbenv/bin/rbenv exec bundle exec rake" + assert_equal map[:rake], "/home/deployer/.rbenv/bin/rbenv exec bundle exec rake" end def test_prefix_initialization_is_thread_safe