diff --git a/ansible/requirements.yml b/ansible/requirements.yml new file mode 100644 index 0000000..0992b91 --- /dev/null +++ b/ansible/requirements.yml @@ -0,0 +1,2 @@ +collections: + - community.docker \ No newline at end of file diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml index 87c1ff9..30ce504 100644 --- a/ansible/roles/common/tasks/main.yml +++ b/ansible/roles/common/tasks/main.yml @@ -20,6 +20,11 @@ name: gnupg update_cache: yes +- name: install jmespath + apt: + name: python3-jmespath + update_cache: yes + - name: set login banner copy: src: issue @@ -35,6 +40,3 @@ owner: root group: root mode: 0644 - -- name: install required pip modules - shell: pip install jmespath diff --git a/ansible/roles/rclone/defaults/main.yml b/ansible/roles/rclone/defaults/main.yml index 52931dd..f6d6d85 100644 --- a/ansible/roles/rclone/defaults/main.yml +++ b/ansible/roles/rclone/defaults/main.yml @@ -1 +1,3 @@ -rclone_directory: "/opt/rclone" \ No newline at end of file +rclone_directory: "/opt/rclone" + +surf_research_vault_documentation: "https://servicedesk.surf.nl/wiki/x/yplsB" diff --git a/ansible/roles/rclone/files/index.php b/ansible/roles/rclone/files/index.php deleted file mode 100644 index 932fcd2..0000000 --- a/ansible/roles/rclone/files/index.php +++ /dev/null @@ -1,36 +0,0 @@ -Links"; - -echo "Here you find the links supported at this moment"; - -echo ""; - -echo "

Authentication

"; - -echo "The following rules for authentication apply:"; - -echo "

Admininistration portal

"; -echo "Federated authentication via your institute at which you are linked to this SRAM Service"; -echo "
"; -echo "You also need to be member of the admins groups that is registered with this SRAM Service."; - -echo "

WebDAV links

"; -echo "Authenticate with your SRAM User ID and your SRAM Token that you have registered for this SRAM Service"; - -// phpinfo(); -?> \ No newline at end of file diff --git a/ansible/roles/rclone/files/mount.conf b/ansible/roles/rclone/files/mount.conf index 22d0708..5e030bd 100644 --- a/ansible/roles/rclone/files/mount.conf +++ b/ansible/roles/rclone/files/mount.conf @@ -9,6 +9,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto https; + proxy_set_header Remote-User $uid; location / { root /var/www/html; diff --git a/ansible/roles/rclone/tasks/main.yml b/ansible/roles/rclone/tasks/main.yml index a3ea263..799a539 100644 --- a/ansible/roles/rclone/tasks/main.yml +++ b/ansible/roles/rclone/tasks/main.yml @@ -44,16 +44,16 @@ src: oidc.conf.j2 dest: "{{ rclone_directory }}/oidc.conf" +- name: copy index.php + template: + src: index.php.j2 + dest: "{{ rclone_directory }}/index.php" + - name: copy mount.conf copy: src: files/mount.conf dest: "{{ rclone_directory }}/mount.conf" -- name: copy index.php - copy: - src: files/index.php - dest: "{{ rclone_directory }}/index.php" - - name: Stop rclone community.docker.docker_compose_v2: project_src: "{{ rclone_directory }}" diff --git a/ansible/roles/rclone/templates/index.php.j2 b/ansible/roles/rclone/templates/index.php.j2 new file mode 100644 index 0000000..a26b228 --- /dev/null +++ b/ansible/roles/rclone/templates/index.php.j2 @@ -0,0 +1,47 @@ +Research Vault"; + +echo "Research Vault documentation
"; + +echo "

Research Vault server addresses

"; + +$message = "No addresses configured yet. Administrator needs to do that first."; + +echo ""; + +if ($message != "") { + echo $message; +} else { + echo "

Authentication

"; + echo "Authenticate with your SRAM User ID and your SRAM Token that you have registered for this SRAM Service"; +} + +echo "

Administration

"; + +echo "You need to be member of the {{ ADMIN_GROUP }}"; + +echo ""; + +// phpinfo(); +?> \ No newline at end of file diff --git a/ansible/roles/vault/tasks/main.yml b/ansible/roles/vault/tasks/main.yml index 1f13268..34c7b2e 100644 --- a/ansible/roles/vault/tasks/main.yml +++ b/ansible/roles/vault/tasks/main.yml @@ -80,6 +80,7 @@ - name: Check Vault health... uri: url: "https://{{ inventory_hostname }}/v1/sys/health" + validate_certs: false register: health - name: Show health... @@ -93,6 +94,7 @@ body: "{{ lookup('file','admin-policy.json') }}" body_format: json status_code: 204 + validate_certs: false headers: Content-Type: application/json X-Vault-Token: "{{ vault_token }}" @@ -108,6 +110,7 @@ body: '{ "password": "{{ admin_password }}", "policies": "admin,default"}' body_format: json status_code: 204 + validate_certs: false headers: Content-Type: application/json X-Vault-Token: "{{ vault_token }}" diff --git a/ansible/vars/default.yml b/ansible/vars/default.yml index 156a5d0..c36dcbc 100644 --- a/ansible/vars/default.yml +++ b/ansible/vars/default.yml @@ -2,15 +2,20 @@ admin_username: admin log_level: ERROR +sram_urn_prefix: "urn:mace:surf.nl:sram:group" + +ADMIN_GROUP: '{{ lookup("ansible.builtin.env", "ADMIN_GROUP", default="") }}' +USERS_GROUP: '{{ lookup("ansible.builtin.env", "USERS_GROUP", default="*") }}' + SRAM_URL: '{{ lookup("ansible.builtin.env", "SRAM_URL", default="https://sram.surf.nl") }}' SRAM_OIDC_BASE_URL: '{{ lookup("ansible.builtin.env", "SRAM_OIDC_BASE_URL", default="https://proxy.sram.surf.nl") }}' SRAM_OIDC_CLIENT_ID: '{{ lookup("ansible.builtin.env", "SRAM_OIDC_CLIENT_ID", default="") }}' SRAM_OIDC_CLIENT_SECRET: '{{ lookup("ansible.builtin.env", "SRAM_OIDC_CLIENT_SECRET", default="") }}' -SRAM_ADMIN_ACCESS_GROUP: '{{ lookup("ansible.builtin.env", "SRAM_ADMIN_ACCESS_GROUP", default="urn:mace:surf.nl:sram:group:...") }}' SRAM_SERVICE_BEARER_TOKEN: '{{ lookup("ansible.builtin.env", "SRAM_SERVICE_BEARER_TOKEN", default="") }}' PROXY_ADMIN_PASSWORD: '{{ lookup("ansible.builtin.env", "PROXY_ADMIN_PASSWORD", default="admin") }}' -PAM_VALIDATE_USERS_ENTITLEMENT: "lambda mount: 'urn:mace:surf.nl:sram:group:{}'.format(mount.replace('-',':'))" +SRAM_ADMIN_ACCESS_GROUP: '{{ sram_urn_prefix }}:{{ ADMIN_GROUP }}' +PAM_VALIDATE_USERS_ENTITLEMENT: '{{ sram_urn_prefix }}:{{ USERS_GROUP }}' diff --git a/rclone/app/api/config/__init__.py b/rclone/app/api/config/__init__.py index aa16a8d..352b004 100644 --- a/rclone/app/api/config/__init__.py +++ b/rclone/app/api/config/__init__.py @@ -3,26 +3,118 @@ import settings import werkzeug import uuid +import json -from flask import request, send_from_directory +from io import BytesIO +from flask import request, send_file, send_from_directory from flask_restplus import Resource, fields, reqparse from api import api, token_required, remote_user_required from vault import rClone -log = logging.getLogger(__name__) +from api.config.crypto import encrypt, decrypt + +log = logging.getLogger() ns = api.namespace('config', description='Operations related to services') my_rclone = rClone() file_upload = reqparse.RequestParser() -file_upload.add_argument('rclone_config_file', - type=werkzeug.datastructures.FileStorage, - location='files', - required=True, - help='Config file') +file_upload.add_argument( + 'rclone_config_file', + type=werkzeug.datastructures.FileStorage, + location='files', + required=True, + help='Config file' +) + +crypted_data = reqparse.RequestParser() +crypted_data.add_argument( + 'crypted_data', + type=str, + required=True, + help='enter crypted data' +) + + +@ns.route('/recover') +class Recover(Resource): + + def get(self): + """ + Returns crypted configuration. + """ + result = {} + + try: + me = request.headers['Remote-User'].replace('"','') + + passphrase = my_rclone.passphrase(me, reset=False)['passphrase'] + + data = {} + for mount in my_rclone.dump().keys(): + config = { + 'name': mount, + 'config': my_rclone.read_rclone_private_config(mount) + } + + data[mount] = encrypt( + passphrase.encode(), + json.dumps(config).encode() + ) + + return data + except Exception as e: + return str(e), 400 + + return result + + @api.expect(crypted_data) + def put(self): + """ + Returns decrypted data of given crypted input. + """ + try: + args = crypted_data.parse_args() + + me = request.headers['Remote-User'].replace('"','') + + passphrase = my_rclone.passphrase(me, reset=False)['passphrase'] + + data = json.loads( + decrypt(passphrase.encode(), args['crypted_data']) + ) + + return send_file(BytesIO(data['config'].encode()), attachment_filename=data['name']+'.conf', as_attachment=True ) + except Exception as e: + return str(e), 401 + +@ns.route('/passphrase') +class PassPhrase(Resource): + + def get(self): + """ + Get my passphrase from Vault + """ + try: + admin = request.headers['Remote-User'].replace('"', '') + + return my_rclone.passphrase(admin, reset=False) + except: + return {} + + def put(self): + """ + (Re-)Set my passphrase in Vault + """ + try: + admin = request.headers['Remote-User'].replace('"', '') + + return my_rclone.passphrase(admin, reset=True) + except: + return {} @ns.route('/dump') @@ -91,6 +183,7 @@ def delete(self, mount): 'parameters': fields.Raw(required=True, description='Mount Parameters') }) + @ns.route('/get', methods=['POST']) class get(Mount): @@ -117,6 +210,7 @@ def post(self): return super().post(mount, payload={ 'config' : config }) + @ns.route('/update', methods=['POST']) class update(Mount): @@ -131,6 +225,7 @@ def post(self): return super().post(mount, payload={ 'config' : config }) + @ns.route('/delete', methods=['POST']) class delete(Mount): @@ -142,6 +237,7 @@ def post(self): return self.delete(api.payload['name']) + @ns.route('/listremotes',methods=['POST']) class ListRemotes(Config): @@ -151,6 +247,7 @@ def post(self): """ return { "remotes": list(super().post().keys()) } + @ns.route('/export',methods=['GET']) class Export(Config): @@ -169,6 +266,7 @@ def get(self): finally: os.remove(dir+'/'+file) + @ns.route('/import', methods=['POST']) class Import(ListRemotes): diff --git a/rclone/app/api/config/crypto.py b/rclone/app/api/config/crypto.py new file mode 100644 index 0000000..88a2f91 --- /dev/null +++ b/rclone/app/api/config/crypto.py @@ -0,0 +1,25 @@ +from Crypto.Cipher import AES +from Crypto.Hash import SHA256 +from Crypto import Random + +def encrypt(key, source, encode=True): + key = SHA256.new(key).digest() # use SHA-256 over our key to get a proper-sized AES key + IV = Random.new().read(AES.block_size) # generate IV + encryptor = AES.new(key, AES.MODE_CBC, IV) + padding = AES.block_size - len(source) % AES.block_size # calculate needed padding + source += bytes([padding]) * padding # Python 2.x: source += chr(padding) * padding + data = IV + encryptor.encrypt(source) # store the IV at the beginning and encrypt + return base64.b64encode(data).decode() if encode else data + +def decrypt(key, source, decode=True): + if decode: + source = base64.b64decode(source.encode()) + key = SHA256.new(key).digest() # use SHA-256 over our key to get a proper-sized AES key + IV = source[:AES.block_size] # extract the IV from the beginning + decryptor = AES.new(key, AES.MODE_CBC, IV) + data = decryptor.decrypt(source[AES.block_size:]) # decrypt + padding = data[-1] # pick the padding value from the end; Python 2.x: ord(data[-1]) + if data[-padding:] != bytes([padding]) * padding: # Python 2.x: chr(padding) * padding + raise ValueError("Invalid padding...") + return data[:-padding] # remove the padding +import base64 diff --git a/rclone/app/app.py b/rclone/app/app.py index d0c67c4..dd79a61 100644 --- a/rclone/app/app.py +++ b/rclone/app/app.py @@ -4,7 +4,7 @@ import logging.config import traceback -from flask import Flask, Blueprint, url_for +from flask import Flask, Blueprint, url_for, request from flask_cors import CORS from functools import wraps @@ -44,6 +44,11 @@ def has_no_empty_params(rule): return len(defaults) >= len(arguments) +@app.before_request +def before_request(): + for k,v in request.headers.items(): + log.debug(f"[APP] Header: {k} --> {v}") + @app.after_request def after_request(response): response.headers.add('Access-Control-Allow-Origin', '*') diff --git a/rclone/app/requirements.txt b/rclone/app/requirements.txt index 5bd6eb4..df4c32f 100644 --- a/rclone/app/requirements.txt +++ b/rclone/app/requirements.txt @@ -5,4 +5,5 @@ requests flask-cors Flask pika -markupsafe==2.0.1 \ No newline at end of file +markupsafe==2.0.1 +pycrypto diff --git a/rclone/app/vault/__init__.py b/rclone/app/vault/__init__.py index afe4c12..628652e 100644 --- a/rclone/app/vault/__init__.py +++ b/rclone/app/vault/__init__.py @@ -232,6 +232,45 @@ def write(self, name, config): def dump(self): return self.mounts() + def passphrase(self, admin, reset=False): + if (reset): + + payload={ 'passphrase': str(uuid.uuid4()) } + + (rc, _) = self.api( + "/v1/secret/data/admin/{}".format(admin), + method="POST", + payload={ 'data': payload } + ) + else: + (rc, data) = self.api( + "/v1/secret/data/admin/{}".format(admin), + method='GET' + ) + payload = json.loads(data)['data']['data'] + + if rc != 200: + raise Exception("status code: {}".format(rc)) + + return payload + + def get_passphrases(self): + result = {} + + (rc, data) = self.api( + "/v1/secret/metadata/admin", + method="LIST" + ) + + if rc == 200: + for admin in json.loads(data)['data']['keys']: + try: + result[admin] = self.passphrase(admin, reset=False)['passphrase'] + except: + pass + + return result + def get_config(self, filename): config = configparser.ConfigParser() @@ -250,6 +289,43 @@ def put_config(self, filename): for name in config.sections(): self.write(name, config[name]) + def write_rclone_private_config(self, name): + try: + details = self.read(name, secrets=True) + except: + self.stop(name) + return + + config = configparser.ConfigParser() + + try: + secrets = details.pop('secrets', None) + + config[name+'_src'] = details + + config[name] = { + 'type': 'crypt', + 'remote': name+'_src:'+secrets['path'], + 'password': secrets['pass'], + 'filename_encryption': 'off', + 'directory_name_encryption': 'false' + } + except Exception as e: + log.error("Error during config: {}: {}".format(name, str(e))) + return + + with open(settings.USERS_CONFIG_PATH+'/'+name+'.conf', 'w') as f: + config.write(f) + + def read_rclone_private_config(self, name): + + self.write_rclone_private_config(name) + + with open(settings.USERS_CONFIG_PATH+'/'+name+'.conf', 'r') as f: + return f.read() + + raise Exception(f"Config: {name} does not exist") + def md5_mount_filename(self, name): return '/usr/local/etc/'+name+'.md5' @@ -285,19 +361,19 @@ def stop_mount(self, name): log.info('Stopping process: {}'.format(pid)) run(['kill', '{}'.format(pid)]) - except: - pass + except Exception as e: + log.error("Error stopping mount: {}: {}".format(name, str(e))) try: os.remove(self.md5_mount_filename(name)) - except OSError: - pass + except Exception as e: + log.error("Error remove md5 hash: {}: {}".format(name, str(e))) try: os.remove(self.pam_mount_filename(name)) - except OSError: - pass - + except Exception as e: + log.error("Error remove pam config: {}: {}".format(name, str(e))) + try: os.remove(self.web_mount_filename(name)) # make sure pending session are rerouted... @@ -309,39 +385,14 @@ def stop_mount(self, name): }} """ ) - except OSError: - pass + except Exception as e: + log.error("Error remove web config: {}: {}".format(name, str(e))) self.flush_config() def start_mount(self, name): - try: - details = self.read(name, secrets=True) - except: - self.stop(name) - return - - config = configparser.ConfigParser() - - try: - secrets = details.pop('secrets', None) - - config[name+'_src'] = details - - config[name] = { - 'type': 'crypt', - 'remote': name+'_src:'+secrets['path'], - 'password': secrets['pass'], - 'filename_encryption': 'off', - 'directory_name_encryption': 'false' - } - except Exception as e: - log.error("Error during config: {}: {}".format(name, str(e))) - return - - with open(settings.USERS_CONFIG_PATH+'/'+name+'.conf', 'w') as f: - config.write(f) + self.write_rclone_private_config(name) md5_file = self.md5_mount_filename(name) md5_new = self.md5_mount_digest(name)