Merge branch 'develop'

.gitignore

@@ -11,3 +11,6 @@ INSTALL*.txt
 RELEASENOTES.txt
 doc/_build
 other_stuff/*/*.png
+
+# roles & dependencies
+roles/marvel-nccr*

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## Quantum Mobile v18.06.0
+
+### Software updates
+* aiida-core v0.12.0
+* aiida-quantumespresso v2.0.1
+
+### Improvements
+
+- Add /scratch directory
+- Using shared folders no longer requires sudo
+
+### Build process
+
+- Separate roles into independent repositories
+- Roles are installed via ansible-galaxy
+- Continuous integration tests for individual roles 
+
 ## Quantum Mobile v18.04.0
 
 ### Software updates

README.md

@@ -1,5 +1,7 @@
 # Quantum Mobile
 
+## What is Quantum Mobile
+
 *Quantum Mobile* is a Virtual Machine for computational materials science.
 
 It comes with a collection of software packages for quantum
@@ -16,37 +18,51 @@ all of which are set up and ready to be used through the
 [AiiDA](http://www.aiida.net) python framework for automated workflows and
 provenance tracking.
 
+You can run Quantum Mobile on your Windows, MacOS or Linux computer using the
+[VirtualBox](http://virtualbox.org/) software.
+You can also install Quantum Mobile on virtual machines using cloud services
+like Amazon Web Services or OpenStack.
+
+![Demo](https://image.ibb.co/n50SdT/quantum_mobile.gif "A brief impression of the Quantum Mobile interface.")
+
 ## Download the VM
 
-Please see [releases](https://github.com/marvel-nccr/quantum-mobile/releases) for the latest VM image and installation instructions.
+Please see [releases](https://github.com/marvel-nccr/quantum-mobile/releases)
+for the latest VM image and installation instructions.
 
-For issues encountered during installation of the VM, see the [FAQ](https://github.com/marvel-nccr/quantum-mobile/wiki/Frequently-Asked-Questions).
+For issues encountered during installation of the VM, see the
+[FAQ](https://github.com/marvel-nccr/quantum-mobile/wiki/Frequently-Asked-Questions).
 
 ## Build it from scratch
 
 You would like to add/remove some components of the VM
 and produce your own modified VM image?
 
-This git repository contains all the vagrant and ansible scripts 
-required to set up the VM from scratch (note: [plan >1h](other_stuff/timings.txt) for this).
+This git repository contains all the vagrant and ansible scripts required to
+set up the VM from scratch (note: [plan >1h](other_stuff/timings.txt) for
+this).
 
 ### Prerequisites
 
 - [vagrant](https://www.vagrantup.com/downloads.html) >= 2.0.1
 - [virtualbox](https://www.virtualbox.org/wiki/Downloads)
 - [python](https://www.python.org/)
-- Host OS: Building VM tested only on Unix systems so far (MacOS, Ubuntu). Might work under Windows with a few modifications.
+- Host OS: Building VM tested only on Unix systems so far (MacOS, Ubuntu).
+  Might work under Windows with a few modifications.
 
 ### Create Virtual Machine
 
 ```
 git checkout git@github.com:marvel-nccr/quantum-mobile.git
 cd quantum-mobile
 pip install -r requirements.txt
+ansible-galaxy install -r requirements.yml
 vagrant plugin install vagrant-vbguest  # optional, improves interface
 vagrant up  # build vm from scratch (takes some tens of minutes)
 ```
 
+Note: Due to a bug in the vbguest vagrant plugin, there are some additional steps #60
+
 ### Create image
 ```
 # optional: reduce size of VM

Vagrantfile

@@ -45,6 +45,7 @@ Vagrant.configure(2) do |config|
   #config.vbguest.no_remote = true
 
   config.vm.box = "bento/ubuntu-16.04"
+  #config.vm.box = "bento/ubuntu-18.04"
   #config.vm.box_version = "== 201801.02.0"
   #config.vm.box = "ubuntu/xenial64"
   config.vm.boot_timeout = 60
@@ -55,42 +56,20 @@ Vagrant.configure(2) do |config|
 
   # Shared folder
   # Unfortunately, VirtualBox only allows to share absolute paths, which cannot
-  # work across all host OS. Until this changes, let's add the shared folder
+  # work across all host OS.
   # https://www.virtualbox.org/ticket/15305
   #config.vm.synced_folder ".", gconfig['vm_shared_folder'], owner: gconfig['vm_user']
 
   # Disable the default shared folder of vagrant
   config.vm.synced_folder ".", "/vagrant", disabled: true
 
-  # provisioner: python needed for ansible provisioner
-  config.vm.provision "bootstrap", type: "shell" do |s|
-    s.inline = "apt-get update && apt-get install -y python2.7 python3"
-  end
-
-  # provisioner: add custom user for ansible provisioner
-  user = gconfig['vm_user']
-  password = gconfig['vm_password']
-  commands = <<-EOF
-if [ ! -d /home/#{user} ] ; then
-  useradd -m -s /bin/bash --groups sudo,adm #{user} && \
-  cp -pr /home/vagrant/.ssh /home/#{user}/ && \
-  chown -R #{user}:#{user} /home/#{user} \
-#  echo #{user}:#{password} | chpasswd 
-fi
-EOF
-  config.vm.provision "adduser", type: "shell" do |s|
-    s.inline = commands
-  end
-
   # provisioner: set up VM via ansible. To (re-)run this step:
   #   vagrant provision --provision-with ansible
   config.vm.provision "ansible" do |ansible|
     ansible.verbose = "v"
-    ansible.inventory_path = './hosts'
     ansible.playbook = "playbook.yml"
     ansible.extra_vars = {
        ansible_python_interpreter: "/usr/bin/python2.7",
-       ansible_user: user
     }
     ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS']
   end

ansible.cfg

@@ -2,8 +2,17 @@
 [defaults]
 log_path = ./ansible.log
 inventory = ./hosts
+roles_path = ./roles
+
+# workaround for https://github.com/ansible/ansible/issues/14426
+timeout = 30
 
 # Use the YAML callback plugin.
 stdout_callback = yaml
 # Use the stdout_callback when running ad-hoc commands.
 bin_ansible_callbacks = True
+
+# when changing user via su, this opens a login shell
+# (i.e. executes .profile)
+# See https://askubuntu.com/a/376386/715125
+ansible_su_flags = "- "

globalconfig.yml

@@ -1,12 +1,12 @@
 ---
 # Caution: This file is read by vagrant, ansible and bash
-vm_version: "18.04.0"
+vm_version: "18.06.0"
 vm_name: "Quantum Mobile"
 vm_description: "A Virtual Machine for Computational Materials Science"
 vm_url: "https://github.com/marvel-nccr/marvel-virtualmachine"
 vm_author: "MARVEL NCCR and MaX CoE"
 vm_author_url: "http://nccr-marvel.ch"
-vm_release_notes_file: "{{ ansible_env.HOME }}/Desktop/RELEASENOTES.txt"
+vm_release_notes_file: "${HOME}/Desktop/RELEASENOTES.txt"
 
 # VM configuration
 vm_hostname: "qmobile"
@@ -16,7 +16,7 @@ vm_memory: 1024
 vm_cpus: 2
 vm_vram: 128
 vm_shared_folder: "/shared"
-vm_codes_folder: "{{ ansible_env.HOME }}/codes"
-vm_examples_folder: "{{ ansible_env.HOME }}/examples"
+vm_codes_folder: "${HOME}/codes"
+vm_examples_folder: "${HOME}/examples"
 vm_headless: False
 vm_browser: chromium # 'chromium' or 'firefox'

host_vars/aws_aiida.yml

@@ -0,0 +1,5 @@
+ansible_ssh_common_args: -i ./keys/aiida_tutorial_aiidaaccount -o StrictHostKeyChecking=no
+
+ansible_become_user: root
+ansible_become_method: su
+ansible_become_pass: "{{ lookup('file', './keys/aws_root_pass') }}" 

host_vars/aws_ubuntu.yml

@@ -0,0 +1 @@
+ansible_ssh_common_args: -i ./keys/aiida-tutorial.pem -o StrictHostKeyChecking=no

hosts

@@ -0,0 +1,4 @@
+[vms:vars]
+ansible_ssh_common_args= -F vagrant-ssh
+[vms]
+default ansible_user=vagrant

keys/.gitignore

@@ -0,0 +1 @@
+*

playbook.yml

@@ -1,9 +1,13 @@
-- name: set up the VM
-  hosts: vms
-  vars:
-    - release_notes: True
-    - run_tests: True
-    - clean: False
+---
+- name: Install ansible prerequisites
+  hosts: default
+  gather_facts: no
+  roles:
+    - marvel-nccr.ansible-prerequisites
+
+- name: Add vm user
+  hosts: default
+  gather_facts: no
   pre_tasks:
     - name: import global variables
       tags: always
@@ -15,56 +19,96 @@
     - name: testing ansible environment
       debug:
         msg: Connecting to host '{{ inventory_hostname }}' as user '{{ ansible_user }}'
+  roles:
+    - name: add user {{ vm_user }} with key
+      role: marvel-nccr.add-user
+      vars:
+        add_user_name: "{{ vm_user }}"
+        add_user_password: "{{ vm_password }}"
+        add_user_sudo: True
+
+# Note: it is important to pass variables to roles within a 'vars' group
+# If they are passed on the same indentation level as the role, the variable is
+# set correctly inside the role, but following roles will read the *default*
+# value of the variable (as defined inside the role).
+- name: set up the VM
+  hosts: default
+  become: True
+  become_user: "{{ vm_user }}"
+  pre_tasks:
+    - name: import global variables
+      tags: always
+      include_vars:
+        file: globalconfig.yml
+  vars:
+    - release_notes: True
+    - run_tests: False
+    - clean: False
+    - root_user: root
+    - cloud_platform: virtualbox
   roles: 
-    - role: common
-      tags: common
-      common_hostname: "{{ vm_hostname }}"
-      common_vm_user: "{{ vm_user }}"
-      common_vm_password: "{{ vm_password }}"
-      common_headless: "{{ vm_headless }}"
-      common_browser: "{{ vm_browser }}"
-    - role: customizations
+    - role: marvel-nccr.simulationbase
+      tags: simulationbase
+      vars:
+        simulationbase_vm_user: "{{ vm_user }}"
+        simulationbase_vm_user_public_key: "{{ lookup('file', './keys/aiida_tutorial_aiidaaccount.pub') }}"
+        simulationbase_hostname: "{{ vm_hostname }}"
+        simulationbase_codes_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.ubuntu-desktop
+      tags: ubuntu_desktop
+      vars:
+        ubuntu_desktop_browser: "{{ vm_browser }}"
+        ubuntu_desktop_vm_user: "{{ vm_user }}"
+      when: not vm_headless
+    - role: marvel-nccr.quantum-mobile-customizations
       tags: customizations
-      customizations_vm_user: "{{ vm_user }}"
-      customizations_vm_password: "{{ vm_password }}"
-      customizations_vm_name: "{{ vm_name }}"
-      customizations_vm_version: "{{ vm_version }}"
-      customizations_vm_shared_folder: "{{ vm_shared_folder }}"
-      customizations_vm_author: "{{ vm_author }}"
-      customizations_headless: "{{ vm_headless }}"
-      customizations_codes_folder: "{{ vm_codes_folder }}"
-      customizations_examples_folder: "{{ vm_examples_folder }}"
-      customizations_browser: "{{ vm_browser }}"
-    - role: editors
+      vars:
+        qm_customizations_vm_user: "{{ vm_user }}"
+        qm_customizations_vm_password: "{{ vm_password }}"
+        qm_customizations_vm_name: "{{ vm_name }}"
+        qm_customizations_vm_version: "{{ vm_version }}"
+        qm_customizations_vm_author: "{{ vm_author }}"
+        qm_customizations_headless: "{{ vm_headless }}"
+        qm_customizations_codes_folder: "{{ vm_codes_folder }}"
+        qm_customizations_examples_folder: "{{ vm_examples_folder }}"
+    - role: marvel-nccr.editors
       tags: editors
-    - role: scheduler
-      tags: scheduler
-      scheduler_hostname: "{{ vm_hostname }}"
-      scheduler_cpus: "{{ vm_cpus }}"
-    - role: simulationtools
-      tags: simulationtools
-    - role: quantum_espresso
+    - role: marvel-nccr.torque
+      tags: torque
+      vars:
+        torque_hostname: "{{ vm_hostname }}"
+        torque_cpus: "{{ vm_cpus }}"
+    - role: marvel-nccr.quantum-espresso
       tags: quantum_espresso
-      quantum_espresso_code_folder: "{{ vm_codes_folder }}"
-    - role: yambo
+      vars: 
+        quantum_espresso_code_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.yambo
       tags: yambo
-      yambo_code_folder: "{{ vm_codes_folder }}"
-    - role: fleur
+      vars: 
+        yambo_code_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.fleur
       tags: fleur
-      fleur_code_folder: "{{ vm_codes_folder }}"
-    - role: siesta
+      vars: 
+        fleur_code_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.siesta
       tags: siesta
-      siesta_code_folder: "{{ vm_codes_folder }}"
-    - role: cp2k
+      vars: 
+        siesta_code_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.cp2k
       tags: cp2k
-      cp2k_code_folder: "{{ vm_codes_folder }}"
-    - role: wannier90
+      vars: 
+        cp2k_code_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.wannier90
       tags: wannier90
-      wannier90_code_folder: "{{ vm_codes_folder }}"
-    - role: aiida
-      tags: aiida
-      aiida_code_folder: "{{ vm_codes_folder }}"
-      aiida_localhost_cpus: "{{ vm_cpus }}"
-      aiida_examples_folder: "{{ vm_examples_folder }}"
-    - role: jupyter
-      tags: jupyter
+      vars: 
+        wannier90_code_folder: "{{ vm_codes_folder }}"
+    - role: marvel-nccr.aiida
+      tags: aiida 
+      vars:
+        aiida_code_folder: "{{ vm_codes_folder }}"
+        aiida_localhost_cpus: "{{ vm_cpus }}"
+        aiida_examples_folder: "{{ vm_examples_folder }}"
+    - role: marvel-nccr.aiidalab
+      tags: aiidalab
+      vars:
+        aiidalab_headless: "{{ vm_headless }}"