Support multiple repos, custom image srcs, encrypted AMI's, various fixes

lib/linecook/builder/darwin_backend.rb

@@ -4,7 +4,6 @@
 #  - create cache loopback image
 #   - dd, based on config file.
 
-
 module Linecook
   module OSXBuilder
     extend self
@@ -71,7 +70,7 @@ def launch_guest
       @pid = guest.pid
       unless @ip
         @guest.kill
-        fail 'Could not acquire ip' 
+        fail 'Could not acquire ip'
       end
       puts "Network acquired, IP is #{@ip}"
       save_run_spec
@@ -80,7 +79,7 @@ def launch_guest
 
     # get and mount the iso
     def get_iso
-      @image_path = Linecook::ImageManager.fetch(:live_iso, profile: :public)
+      @image_path = Linecook::ImageManager.fetch(:live_iso, profile: Linecook.config[:image][:images][:live_iso][:profile])
     end
 
     def mount_iso

lib/linecook/builder/lxc.rb

@@ -197,7 +197,7 @@ def setup_bridge
       end
 
       def setup_image
-        @source_path = Linecook::ImageManager.fetch(@source_image, profile: :public)
+        @source_path = Linecook::ImageManager.fetch(@source_image, profile: Linecook.config[:image][:images][:live_iso][:profile])
         if @remote
           name = File.basename(@source_path)
           dest = "/u/linecook/images/#{name}"

lib/linecook/builder/manager.rb

@@ -57,7 +57,7 @@ def backend_for_platform
     def increase_loop_devices
       kparams = {}
 
-      ssh.capture('cat /proc/cmdline').split(/\s+/).each do |param|
+      ssh.capture('sudo cat /proc/cmdline').split(/\s+/).each do |param|
         k,v = param.split('=')
         kparams[k] = v
       end

lib/linecook/cli.rb

@@ -146,6 +146,7 @@ class Linecook::CLI < Thor
   method_option :clean, type: :boolean, default: false, desc: 'Clean up all build artifacts', aliases: '-c'
   method_option :build, type: :boolean, default: true, desc: 'Build the image', aliases: '-b'
   method_option :snapshot, type: :boolean, default: false, desc: 'Snapshot the resulting build to create an image', aliases: '-s'
+  method_option :ami, type: :boolean, default: false, desc: 'Create an ami. Implies --snapshot.', aliases: '-a'
   method_option :upload, type: :boolean, default: false, desc: 'Upload the resulting build. Implies --snapshot and --encrypt.', aliases: '-u'
   method_option :package, type: :boolean, default: false, desc: 'Package the resulting image. Implies --upload, --snapshot and --encrypt.', aliases: '-p'
   def bake

lib/linecook/image/s3.rb

@@ -54,7 +54,7 @@ def list_objects(type: nil)
     def client
       @client ||= begin
         Aws.config[:credentials] = Aws::Credentials.new(Linecook.config[:aws][:access_key], Linecook.config[:aws][:secret_key])
-        Aws.config[:region] = 'us-east-1'
+        Aws.config[:region] = Linecook.config[:packager][:ebs][:region]
         Aws::S3::Client.new
       end
     end

lib/linecook/packager/ebs.rb

@@ -15,12 +15,13 @@ module Packager
     class EBS
       include Executor
 
-      def initialize(hvm: true, size: 10, region: nil, copy_regions: [], account_ids: [])
+      def initialize(hvm: true, size: 10, region: nil, copy_regions: [], account_ids: [], encrypted_ami: false)
         @hvm = hvm
         @size = size
         @region = region
         @copy_regions = copy_regions
         @account_ids = account_ids
+        @encrypted_ami = encrypted_ami
       end
 
       def package(image, type: nil, ami: nil)
@@ -51,8 +52,9 @@ def finalize
         execute("echo \"UUID=\\\"$(blkid -o value -s UUID #{@rootdev})\\\" /               ext4            defaults  1 2\" > /tmp/fstab")
         execute("mv /tmp/fstab #{@root}/etc/fstab")
         chroot_exec('apt-get update')
-        chroot_exec('apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y --force-yes --no-upgrade install grub-pc grub-legacy-ec2')
+        chroot_exec('bash -c "DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y --force-yes --no-upgrade install grub-pc grub-legacy-ec2"')
         chroot_exec('update-grub')
+        chroot_exec('rm -rf /etc/ssh/ssh_host_*')
         execute("grub-install --root-directory=#{@root} $(echo #{@rootdev} | sed \"s/[0-9]*//g\")") if @hvm
       end
 
@@ -109,7 +111,8 @@ def create_volume
         resp = client.create_volume({
           size: @size,
           availability_zone: availability_zone, # required
-          volume_type: "standard", # accepts standard, io1, gp2
+          volume_type: @hvm ? 'gp2' : 'standard', # accepts standard, io1, gp2
+          encrypted: @encrypted_ami,
         })
 
         @volume_id = resp.volume_id

lib/linecook/provisioner/chef-zero.rb

@@ -17,7 +17,8 @@ def provision(build, role)
         first_boot: {
           run_list: role_config[:run_list]
         },
-        audit: Linecook.config[:provisioner][:chefzero][:audit]
+        audit: Linecook.config[:provisioner][:chefzero][:audit],
+        bootstrap_file: Linecook.config[:chef][:bootstrap_file]
       )
 
       puts "Establishing connection to build..."
@@ -26,7 +27,7 @@ def provision(build, role)
       build.ssh.upload(script, '/tmp/chef_bootstrap')
       build.ssh.run('[ -f /var/chef/cache/chef-client-running.pid ] && sudo rm -f /var/chef/cache/chef-client-running.pid || true')
       build.ssh.run("sudo hostname #{chef_config[:node_name]}")
-      build.ssh.run('sudo bash /tmp/chef_bootstrap')
+      build.ssh.run('sudo bash /tmp/chef_bootstrap 2>&1 | tee /tmp/chef-client.log')
       build.ssh.run('sudo rm -rf /etc/chef')
       build.ssh.stop_forwarding
       Chefdepartie.stop
@@ -40,19 +41,19 @@ def chef_port
     private
 
     def setup
-      ChefProvisioner::Config.setup(client: 'linecook', listen: 'localhost')
+      ChefProvisioner::Config.setup(client: 'linecook', listen: '127.0.0.1')
       config = Linecook.config
 
       chef_config = config[:chef]
-      chef_config.merge!(node_name: "linecook-#{SecureRandom.hex(4)}",
+      chef_config.merge!(node_name: "linecook-#{SecureRandom.hex(4)}.linecook.local",
                          chef_server_url: ChefProvisioner::Config.server)
       Chefdepartie.run(background: true, config: chef_config, cache: Cache.path)
       chef_config
     end
 
     # Required in order to have multiple builds run on different refs
     module Cache
-      CACHE_PATH = File.join(Linecook::Config::LINECOOK_HOME, 'chefcache').freeze
+      CACHE_PATH = File.join(Linecook::Config::LINECOOK_HOME, "#{Digest::SHA1.hexdigest(Dir.pwd)[0...7]}-chefcache").freeze
       PIDFILE = File.join(CACHE_PATH, 'pid')
       STAMPFILE = File.join(CACHE_PATH, 'stamp')
       STALE_THRESHOLD = 86400 # one day in seconds

lib/linecook/provisioner/manager.rb

@@ -9,13 +9,13 @@ module Linecook
   module Baker
     extend self
 
-    def bake(name: nil, tag: nil, id: nil, snapshot: nil, upload: nil, package: nil, build: nil, keep: nil, clean: nil)
+    def bake(name: nil, tag: nil, id: nil, snapshot: nil, upload: nil, package: nil, build: nil, keep: nil, clean: nil, ami: nil)
       build_agent = Linecook::Build.new(name, tag: tag, id: id, image: image(name))
       resume = clean ? false : true
       provider(name).provision(build_agent, name) if build
-      snapshot = build_agent.snapshot(save: true, resume: resume) if snapshot ||  upload || package
+      snapshot = build_agent.snapshot(save: true, resume: resume) if snapshot || upload || package || ami
       Linecook::ImageManager.upload(snapshot, type: build_agent.type) if upload || package
-      Linecook::Packager.package(snapshot, type: build_agent.type) if package
+      Linecook::Packager.package(snapshot, type: build_agent.type, ami: ami) if package || ami
     rescue => e
       puts e.message
       puts e.backtrace

lib/linecook/util/config.rb

@@ -20,6 +20,10 @@ module Config
     LINECOOK_HOME = File.expand_path('~/.linecook').freeze
     DEFAULT_CONFIG_PATH = File.join(LINECOOK_HOME, 'config.yml').freeze
     DEFAULT_CONFIG = {
+      chef: {
+        data_bag_path: ['../data_bags'],
+        chef_repo_path: '..'
+      },
       builder: {
         image: :live_image,
         name: 'builder',
@@ -61,7 +65,8 @@ module Config
           size: 10,
           region: 'us-east-1',
           copy_regions: [],
-          account_ids: []
+          account_ids: [],
+          encrypted_ami: false
         }
       },
       roles: {

linecook.gemspec

@@ -19,8 +19,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'sshkit', ['=1.7.1']
   s.add_runtime_dependency 'sshkey', ['=1.8.0']
   s.add_runtime_dependency 'octokit', ['=4.2.0']
-  s.add_runtime_dependency 'chefdepartie', ['=0.1.1']
-  s.add_runtime_dependency 'chef-provisioner', ['=0.1.4']
+  s.add_runtime_dependency 'chefdepartie', ['>=0.1.1']
+  s.add_runtime_dependency 'chef-provisioner', ['>=0.1.4']
   s.add_runtime_dependency 'activesupport', ['=4.2.5']
   s.add_runtime_dependency 'ruby-progressbar', ['=1.7.5']
   s.add_runtime_dependency 'ipaddress', ['=0.8.0']