From 863f052d37e31fbd26680ab10410dc709e340c62 Mon Sep 17 00:00:00 2001 From: ReimarBauer Date: Fri, 6 Sep 2024 10:47:27 +0200 Subject: [PATCH] renamed and refactored names for mscolab (#2499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit removed the xmlsec binary selector based on CONDA_PREFIX provided separate config for msidp separate dir for mscolab server data and work asynchronously by msolab ui --------- Co-authored-by: Matthias Riße <9308656+matrss@users.noreply.github.com> --- conftest.py | 6 +-- docs/conf_sso_test_msscolab.rst | 6 +++ .../config/mscolab/mscolab_settings.py.sample | 18 +++---- .../mscolab/setup_saml2_backend.py.sample | 10 ++-- .../config/msidp/msidp_settings.py.sample | 36 ++++++++++++++ docs/sso_via_saml_mscolab.rst | 2 +- mslib/mscolab/app/__init__.py | 2 +- mslib/mscolab/conf.py | 32 +++++++------ mslib/mscolab/mscolab.py | 26 +++++----- mslib/mscolab/seed.py | 6 +-- mslib/mscolab/sockets_manager.py | 2 +- mslib/mscolab/utils.py | 4 +- mslib/msidp/conf.py | 48 +++++++++++++++++++ mslib/msidp/idp.py | 4 +- mslib/msidp/idp_conf.py | 32 +++++-------- mslib/msui/mscolab.py | 10 ++-- mslib/msui/msui_mainwindow.py | 13 ++++- tests/_test_mscolab/test_files.py | 4 +- tests/_test_mscolab/test_mscolab.py | 8 ++-- tests/_test_mscolab/test_server.py | 4 +- .../test_server_auth_required.py | 2 +- tests/_test_mscolab/test_utils.py | 2 +- tests/_test_msui/test_mscolab.py | 12 ++--- tests/_test_msui/test_mscolab_admin_window.py | 4 +- .../test_mscolab_merge_waypoints.py | 6 +-- tests/_test_msui/test_mscolab_operation.py | 4 +- .../test_mscolab_version_history.py | 4 +- tests/fixtures.py | 2 +- 28 files changed, 202 insertions(+), 107 deletions(-) create mode 100644 docs/samples/config/msidp/msidp_settings.py.sample create mode 100644 mslib/msidp/conf.py diff --git a/conftest.py b/conftest.py index b9bf7f543..13a0a5528 100644 --- a/conftest.py +++ b/conftest.py @@ -111,9 +111,9 @@ def generate_initial_config(): root_fs.makedir('colabTestData') BASE_DIR = ROOT_DIR DATA_DIR = fs.path.join(ROOT_DIR, 'colabTestData') -# mscolab data directory -MSCOLAB_DATA_DIR = fs.path.join(DATA_DIR, 'filedata') -MSCOLAB_SSO_DIR = fs.path.join(DATA_DIR, 'datasso') +# mscolab data directory for operation git repositories +OPERATIONS_DATA = fs.path.join(DATA_DIR, 'filedata') +SSO_DIR = fs.path.join(DATA_DIR, 'datasso') # In the unit days when Operations get archived because not used ARCHIVE_THRESHOLD = 30 diff --git a/docs/conf_sso_test_msscolab.rst b/docs/conf_sso_test_msscolab.rst index ad9ecaf19..9ecbd4097 100644 --- a/docs/conf_sso_test_msscolab.rst +++ b/docs/conf_sso_test_msscolab.rst @@ -30,7 +30,13 @@ To set up a local identity provider with the mscolab server, you'll first need t ---------------- Before getting started, you should correctly activate the environments, set the correct Python path as explained in the mss instructions : https://github.com/Open-MSS/MSS/tree/develop#readme +Setup your +**msidp_settings.py** + + .. literalinclude:: samples/config/msidp/msidp_settings.py.sample + +.. _configuration-mscolab: 2. Generate Keys, Certificates, and backend_saml files ------------------------------------------------------ diff --git a/docs/samples/config/mscolab/mscolab_settings.py.sample b/docs/samples/config/mscolab/mscolab_settings.py.sample index 7249ee5fd..64eb16e66 100644 --- a/docs/samples/config/mscolab/mscolab_settings.py.sample +++ b/docs/samples/config/mscolab/mscolab_settings.py.sample @@ -44,8 +44,16 @@ BASE_DIR = os.path.abspath(os.path.dirname(__file__)) # Directory in which all data related to Mscolab is stored DATA_DIR = os.path.join(BASE_DIR, "colabdata") -# Where mscolab project files are stored on the server -MSCOLAB_DATA_DIR = os.path.join(DATA_DIR, 'filedata') +# mscolab data directory for operation git repositories +OPERATIONS_DATA = os.path.join(DATA_DIR, 'filedata') + +# SSO by SAML2 is optional + +# dir where mscolab single sign-on process files are stored +SSO_DIR = os.path.join(DATA_DIR, 'datasso') + +# Enable SSL certificate verification during SSO between MSColab and IdP +SSO_SSL_CERT_VERIFICATION = True # Directory where uploaded images and documents in the chat are stored UPLOAD_FOLDER = os.path.join(DATA_DIR, 'uploads') @@ -99,9 +107,3 @@ USE_SAML2 = False # all users in that Group are set to the operations of that category # having the roles in the TexGroup GROUP_POSTFIX = "Group" - -# Enable SSL certificate verification during SSO between MSColab and IdP -ENABLE_SSO_SSL_CERT_VERIFICATION = True - -# dir where mscolab single sign process files are stored -MSCOLAB_SSO_DIR = os.path.join(DATA_DIR, 'datasso') diff --git a/docs/samples/config/mscolab/setup_saml2_backend.py.sample b/docs/samples/config/mscolab/setup_saml2_backend.py.sample index d2bc963e6..941a2a4cc 100644 --- a/docs/samples/config/mscolab/setup_saml2_backend.py.sample +++ b/docs/samples/config/mscolab/setup_saml2_backend.py.sample @@ -22,19 +22,19 @@ class setup_saml2_backend: ] - if os.path.exists(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml"): - with open(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj: + if os.path.exists(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml"): + with open(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj: yaml_data = yaml.safe_load(fobj) # go through configured IDPs and set conf file paths for particular files for configured_idp in CONFIGURED_IDPS: # set CRTs and metadata paths for the localhost_test_idp if 'localhost_test_idp' == configured_idp['idp_identity_name']: yaml_data["config"]["localhost_test_idp"]["key_file"] = \ - f'{mscolab_settings.MSCOLAB_SSO_DIR}/key_mscolab.key' # set path to your mscolab key file + f'{mscolab_settings.SSO_DIR}/key_mscolab.key' # set path to your mscolab key file yaml_data["config"]["localhost_test_idp"]["cert_file"] = \ - f'{mscolab_settings.MSCOLAB_SSO_DIR}/crt_mscolab.crt' # set path to your mscolab certificate file + f'{mscolab_settings.SSO_DIR}/crt_mscolab.crt' # set path to your mscolab certificate file yaml_data["config"]["localhost_test_idp"]["metadata"]["local"][0] = \ - f'{mscolab_settings.MSCOLAB_SSO_DIR}/idp.xml' # set path to your idp metadata xml file + f'{mscolab_settings.SSO_DIR}/idp.xml' # set path to your idp metadata xml file # configuration localhost_test_idp Saml2Client try: diff --git a/docs/samples/config/msidp/msidp_settings.py.sample b/docs/samples/config/msidp/msidp_settings.py.sample new file mode 100644 index 000000000..138e22634 --- /dev/null +++ b/docs/samples/config/msidp/msidp_settings.py.sample @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" + + mslib.msidp.conf.py.example + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + config for msidp + + This file is part of mss. + + :copyright: Copyright 2023 Nilupul Manodya + :copyright: Copyright 2023-2024 by the MSS team, see AUTHORS. + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import os + + +# our default dir for mss content +BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss') + +DATA_DIR = os.path.join(BASE_DIR, "colabdata") + +# dir where mscolab single sign-on process files are stored +SSO_DIR = os.path.join(DATA_DIR, 'datasso') diff --git a/docs/sso_via_saml_mscolab.rst b/docs/sso_via_saml_mscolab.rst index 19f5fabe8..87413c163 100644 --- a/docs/sso_via_saml_mscolab.rst +++ b/docs/sso_via_saml_mscolab.rst @@ -471,7 +471,7 @@ Configuration in MSColab settings for Keycloak Configuration mss_saml2_backend.yaml file - Create your mss_saml2_backend.yaml file in your ``MSCOLAB_SSO_DIR``. + Create your mss_saml2_backend.yaml file in your ``SSO_DIR``. .. code:: text diff --git a/mslib/mscolab/app/__init__.py b/mslib/mscolab/app/__init__.py index 1e9fe719e..7ed623e63 100644 --- a/mslib/mscolab/app/__init__.py +++ b/mslib/mscolab/app/__init__.py @@ -52,7 +52,7 @@ APP.config.from_object(__name__) APP.route = prefix_route(APP.route, SCRIPT_NAME) -APP.config['MSCOLAB_DATA_DIR'] = mscolab_settings.MSCOLAB_DATA_DIR +APP.config['OPERATIONS_DATA'] = mscolab_settings.OPERATIONS_DATA APP.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI APP.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False APP.config['SQLALCHEMY_ECHO'] = mscolab_settings.SQLALCHEMY_ECHO diff --git a/mslib/mscolab/conf.py b/mslib/mscolab/conf.py index 4321f7ab2..e88bc6d57 100644 --- a/mslib/mscolab/conf.py +++ b/mslib/mscolab/conf.py @@ -53,12 +53,20 @@ class default_mscolab_settings: CORS_ORIGINS = ["*"] # dir where msui output files are stored - BASE_DIR = os.path.expanduser("~") + BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss') DATA_DIR = os.path.join(BASE_DIR, "colabdata") - # mscolab data directory - MSCOLAB_DATA_DIR = os.path.join(DATA_DIR, 'filedata') + # mscolab data directory for operation git repositories + OPERATIONS_DATA = os.path.join(DATA_DIR, 'filedata') + + # SSO by SAML2 is optional + + # dir where mscolab single sign-on process files are stored + SSO_DIR = os.path.join(DATA_DIR, 'datasso') + + # Enable SSL certificate verification during SSO between MSColab and IdP + SSO_SSL_CERT_VERIFICATION = True # MYSQL CONNECTION STRING: "mysql+pymysql://:@:/?charset=utf8mb4" SQLALCHEMY_DB_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'mscolab.db') @@ -126,12 +134,6 @@ class default_mscolab_settings: # accounts on a database on the server DIRECT_LOGIN = True - # Enable SSL certificate verification during SSO between MSColab and IdP - ENABLE_SSO_SSL_CERT_VERIFICATION = True - - # dir where mscolab single sign process files are stored - MSCOLAB_SSO_DIR = os.path.join(DATA_DIR, 'datasso') - mscolab_settings = default_mscolab_settings() @@ -165,19 +167,19 @@ class setup_saml2_backend: # } # }, ] - if os.path.exists(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml"): - with open(f"{mscolab_settings.MSCOLAB_SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj: + if os.path.exists(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml"): + with open(f"{mscolab_settings.SSO_DIR}/mss_saml2_backend.yaml", encoding="utf-8") as fobj: yaml_data = yaml.safe_load(fobj) # go through configured IDPs and set conf file paths for particular files for configured_idp in CONFIGURED_IDPS: # set CRTs and metadata paths for the localhost_test_idp if 'localhost_test_idp' == configured_idp['idp_identity_name']: yaml_data["config"]["localhost_test_idp"]["key_file"] = \ - f'{mscolab_settings.MSCOLAB_SSO_DIR}/key_mscolab.key' + f'{mscolab_settings.SSO_DIR}/key_mscolab.key' yaml_data["config"]["localhost_test_idp"]["cert_file"] = \ - f'{mscolab_settings.MSCOLAB_SSO_DIR}/crt_mscolab.crt' + f'{mscolab_settings.SSO_DIR}/crt_mscolab.crt' yaml_data["config"]["localhost_test_idp"]["metadata"]["local"][0] = \ - f'{mscolab_settings.MSCOLAB_SSO_DIR}/idp.xml' + f'{mscolab_settings.SSO_DIR}/idp.xml' # configuration localhost_test_idp Saml2Client try: @@ -187,7 +189,7 @@ class setup_saml2_backend: Ignore this warning when you initialize metadata.") localhost_test_idp = SPConfig().load(yaml_data["config"]["localhost_test_idp"]) - localhost_test_idp.verify_ssl_cert = mscolab_settings.ENABLE_SSO_SSL_CERT_VERIFICATION + localhost_test_idp.verify_ssl_cert = mscolab_settings.SSO_SSL_CERT_VERIFICATION sp_localhost_test_idp = Saml2Client(localhost_test_idp) configured_idp['idp_data']['saml2client'] = sp_localhost_test_idp diff --git a/mslib/mscolab/mscolab.py b/mslib/mscolab/mscolab.py index e3b521872..d802dcbb4 100644 --- a/mslib/mscolab/mscolab.py +++ b/mslib/mscolab/mscolab.py @@ -102,9 +102,9 @@ def handle_mscolab_certificate_init(): try: cmd = ["openssl", "req", "-newkey", "rsa:4096", "-keyout", - os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "key_mscolab.key"), + os.path.join(mscolab_settings.SSO_DIR, "key_mscolab.key"), "-nodes", "-x509", "-days", "365", "-batch", "-subj", - "/CN=localhost", "-out", os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, + "/CN=localhost", "-out", os.path.join(mscolab_settings.SSO_DIR, "crt_mscolab.crt")] subprocess.run(cmd, check=True) logging.info("generated CRTs for the mscolab server.") @@ -119,9 +119,9 @@ def handle_local_idp_certificate_init(): try: cmd = ["openssl", "req", "-newkey", "rsa:4096", "-keyout", - os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "key_local_idp.key"), + os.path.join(mscolab_settings.SSO_DIR, "key_local_idp.key"), "-nodes", "-x509", "-days", "365", "-batch", "-subj", - "/CN=localhost", "-out", os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "crt_local_idp.crt")] + "/CN=localhost", "-out", os.path.join(mscolab_settings.SSO_DIR, "crt_local_idp.crt")] subprocess.run(cmd, check=True) logging.info("generated CRTs for the local identity provider") return True @@ -252,7 +252,7 @@ def handle_mscolab_backend_yaml_init(): # name_id_format_allow_create: true """ try: - file_path = os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "mss_saml2_backend.yaml") + file_path = os.path.join(mscolab_settings.SSO_DIR, "mss_saml2_backend.yaml") with open(file_path, "w", encoding="utf-8") as file: file.write(saml_2_backend_yaml_content) return True @@ -278,7 +278,7 @@ def handle_mscolab_metadata_init(repo_exists): process = subprocess.Popen(command) cmd_curl = ["curl", "--retry", "5", "--retry-connrefused", "--retry-delay", "3", "http://localhost:8083/metadata/localhost_test_idp", - "-o", os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "metadata_sp.xml")] + "-o", os.path.join(mscolab_settings.SSO_DIR, "metadata_sp.xml")] subprocess.run(cmd_curl, check=True) process.terminate() logging.info('mscolab metadata file generated succesfully') @@ -293,8 +293,8 @@ def handle_local_idp_metadata_init(repo_exists): print('generating metadata for localhost identity provider') try: - if os.path.exists(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml")): - os.remove(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml")) + if os.path.exists(os.path.join(mscolab_settings.SSO_DIR, "idp.xml")): + os.remove(os.path.join(mscolab_settings.SSO_DIR, "idp.xml")) idp_conf_path = os.path.join("mslib", "msidp", "idp_conf.py") @@ -305,15 +305,15 @@ def handle_local_idp_metadata_init(repo_exists): cmd = ["make_metadata", idp_conf_path] - with open(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml"), + with open(os.path.join(mscolab_settings.SSO_DIR, "idp.xml"), "w", encoding="utf-8") as output_file: subprocess.run(cmd, stdout=output_file, check=True) logging.info("idp metadata file generated successfully") return True except subprocess.CalledProcessError as error: # Delete the idp.xml file when the subprocess fails - if os.path.exists(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml")): - os.remove(os.path.join(mscolab_settings.MSCOLAB_SSO_DIR, "idp.xml")) + if os.path.exists(os.path.join(mscolab_settings.SSO_DIR, "idp.xml")): + os.remove(os.path.join(mscolab_settings.SSO_DIR, "idp.xml")) print(f"Error while generating metadata for localhost identity provider: {error}") return False @@ -323,8 +323,8 @@ def handle_sso_crts_init(): This will generate necessary CRTs files for sso in mscolab through localhost idp """ print("\n\nmscolab sso conf initiating......") - if os.path.exists(mscolab_settings.MSCOLAB_SSO_DIR): - shutil.rmtree(mscolab_settings.MSCOLAB_SSO_DIR) + if os.path.exists(mscolab_settings.SSO_DIR): + shutil.rmtree(mscolab_settings.SSO_DIR) create_files() if not handle_mscolab_certificate_init(): print('Error while handling mscolab certificate.') diff --git a/mslib/mscolab/seed.py b/mslib/mscolab/seed.py index 8d9f23457..8a740a7dd 100644 --- a/mslib/mscolab/seed.py +++ b/mslib/mscolab/seed.py @@ -62,7 +62,7 @@ def add_all_users_default_operation(path='TEMPLATE', description="Operation to k operation = Operation(path, description) db.session.add(operation) db.session.commit() - with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir: + with fs.open_fs(mscolab_settings.OPERATIONS_DATA) as file_dir: if not file_dir.exists(path): file_dir.makedir(path) file_dir.writetext(f'{path}/main.ftml', mscolab_settings.STUB_CODE) @@ -150,7 +150,7 @@ def add_operation(operation_name, description): operation = Operation(operation_name, description) db.session.add(operation) db.session.commit() - with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir: + with fs.open_fs(mscolab_settings.OPERATIONS_DATA) as file_dir: if not file_dir.exists(operation_name): file_dir.makedir(operation_name) file_dir.writetext(f'{operation_name}/main.ftml', mscolab_settings.STUB_CODE) @@ -396,7 +396,7 @@ def seed_data(): db.session.commit() db.session.close() - with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir: + with fs.open_fs(mscolab_settings.OPERATIONS_DATA) as file_dir: file_paths = ['one', 'two', 'three', 'four', 'Admin_Test', 'test_mscolab'] for file_path in file_paths: file_dir.makedir(file_path) diff --git a/mslib/mscolab/sockets_manager.py b/mslib/mscolab/sockets_manager.py index a51db4199..719d28b45 100644 --- a/mslib/mscolab/sockets_manager.py +++ b/mslib/mscolab/sockets_manager.py @@ -334,7 +334,7 @@ def _setup_managers(app): """ cm = ChatManager() - fm = FileManager(app.config["MSCOLAB_DATA_DIR"]) + fm = FileManager(app.config["OPERATIONS_DATA"]) sm = SocketsManager(cm, fm) # sockets related handlers socketio.on_event('connect', sm.handle_connect) diff --git a/mslib/mscolab/utils.py b/mslib/mscolab/utils.py index bdbf3545b..6f5db86e9 100644 --- a/mslib/mscolab/utils.py +++ b/mslib/mscolab/utils.py @@ -82,6 +82,6 @@ def os_fs_create_dir(directory_path): def create_files(): - os_fs_create_dir(mscolab_settings.MSCOLAB_DATA_DIR) + os_fs_create_dir(mscolab_settings.OPERATIONS_DATA) os_fs_create_dir(mscolab_settings.UPLOAD_FOLDER) - os_fs_create_dir(mscolab_settings.MSCOLAB_SSO_DIR) + os_fs_create_dir(mscolab_settings.SSO_DIR) diff --git a/mslib/msidp/conf.py b/mslib/msidp/conf.py new file mode 100644 index 000000000..751781b44 --- /dev/null +++ b/mslib/msidp/conf.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +""" + + mslib.msidp.conf + ~~~~~~~~~~~~~~~~ + + config for msidp. + + This file is part of MSS. + + :copyright: Copyright 2023 Nilupul Manodya + :copyright: Copyright 2023-2024 by the MSS team, see AUTHORS. + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import os +import logging + + +class default_msidp_settings: + # our default dir for mss content + BASE_DIR = os.path.join(os.path.expanduser("~"), 'mss') + + DATA_DIR = os.path.join(BASE_DIR, "colabdata") + + # dir where mscolab single sign-on process files are stored + SSO_DIR = os.path.join(DATA_DIR, 'datasso') + + +msidp_settings = default_msidp_settings() + +try: + import msidp_settings as user_settings + logging.info("Using user defined settings") + msidp_settings.__dict__.update(user_settings.__dict__) +except ImportError as ex: + logging.warning(u"Couldn't import msidp_settings (ImportError:'%s'), using dummy config.", ex) diff --git a/mslib/msidp/idp.py b/mslib/msidp/idp.py index a6f1f78de..521b36486 100644 --- a/mslib/msidp/idp.py +++ b/mslib/msidp/idp.py @@ -83,7 +83,7 @@ from mslib.msidp.idp_user import EXTRA from mslib.msidp.idp_user import USERS, PASSWD from mako.lookup import TemplateLookup -from mslib.mscolab.conf import mscolab_settings +from mslib.msidp.conf import msidp_settings logger = logging.getLogger("saml2.idp") logger.setLevel(logging.WARNING) @@ -91,7 +91,7 @@ DOCS_SERVER_PATH = os.path.dirname(os.path.abspath(msidp.__file__)) LOOKUP = TemplateLookup( directories=[os.path.join(DOCS_SERVER_PATH, "templates"), os.path.join(DOCS_SERVER_PATH, "htdocs")], - module_directory=os.path.join(mscolab_settings.DATA_DIR, 'msidp_modules'), + module_directory=os.path.join(msidp_settings.DATA_DIR, 'msidp_modules'), input_encoding="utf-8", output_encoding="utf-8", ) diff --git a/mslib/msidp/idp_conf.py b/mslib/msidp/idp_conf.py index 21d726a4e..a545713db 100644 --- a/mslib/msidp/idp_conf.py +++ b/mslib/msidp/idp_conf.py @@ -24,8 +24,7 @@ limitations under the License. """ -# Parts of the code -import logging + import os.path from saml2 import BINDING_HTTP_ARTIFACT @@ -36,28 +35,21 @@ from saml2.saml import NAME_FORMAT_URI from saml2.saml import NAMEID_FORMAT_PERSISTENT from saml2.saml import NAMEID_FORMAT_TRANSIENT +from saml2.sigver import get_xmlsec_binary + +from mslib.msidp.conf import msidp_settings -XMLSEC_PATH = os.path.join(os.environ["CONDA_PREFIX"], "bin", "xmlsec1") -if not os.path.exists(XMLSEC_PATH): - logging.warning("%s not found", XMLSEC_PATH) # CRTs and metadata files can be generated through the mscolab server. # if configured that way CRTs DIRs should be same in both IDP and mscolab server. -BASE_DIR = os.path.expanduser("~") -DATA_DIR = os.path.join(BASE_DIR, "colabdata") -MSCOLAB_SSO_DIR = os.path.join(DATA_DIR, 'datasso') - -BASEDIR = os.path.abspath(os.path.dirname(__file__)) - - -def full_path(local_file): - """Return the full path by joining the BASEDIR and local_file.""" - return os.path.join(BASEDIR, local_file) +BASE_DIR = msidp_settings.BASE_DIR +DATA_DIR = msidp_settings.DATA_DIR +SSO_DIR = msidp_settings.SSO_DIR def sso_dir_path(local_file): - """Return the full path by joining the MSCOLAB_SSO_DIR and local_file.""" - return os.path.join(MSCOLAB_SSO_DIR, local_file) + """Return the full path by joining the SSO_DIR and local_file.""" + return os.path.join(SSO_DIR, local_file) HOST = 'localhost' @@ -71,8 +63,8 @@ def sso_dir_path(local_file): BASE = f"http://{HOST}:{PORT}" # HTTPS cert information -SERVER_CERT = f"{MSCOLAB_SSO_DIR}/crt_local_idp.crt" -SERVER_KEY = f"{MSCOLAB_SSO_DIR}/key_local_idp.key" +SERVER_CERT = f"{SSO_DIR}/crt_local_idp.crt" +SERVER_KEY = f"{SSO_DIR}/key_local_idp.key" CERT_CHAIN = "" SIGN_ALG = None DIGEST_ALG = None @@ -170,7 +162,7 @@ def sso_dir_path(local_file): ], # This database holds the map between a subject's local identifier and # the identifier returned to a SP - "xmlsec_binary": XMLSEC_PATH, + "xmlsec_binary": get_xmlsec_binary(), # "attribute_map_dir": "../attributemaps", "logging": { "version": 1, diff --git a/mslib/msui/mscolab.py b/mslib/msui/mscolab.py index f5e517ddd..4cdf5be64 100644 --- a/mslib/msui/mscolab.py +++ b/mslib/msui/mscolab.py @@ -481,7 +481,7 @@ class MSUIMscolab(QtCore.QObject): signal_permission_revoked = QtCore.pyqtSignal(int) signal_render_new_permission = QtCore.pyqtSignal(int, str) - def __init__(self, parent=None, data_dir=None): + def __init__(self, parent=None, local_operations_data=None): super().__init__(parent) self.ui = parent @@ -580,10 +580,10 @@ def __init__(self, parent=None, data_dir=None): self.lastChangeMessage = "" # set data dir, uri - if data_dir is None: + if local_operations_data is None: self.data_dir = config_loader(dataset="mss_dir") else: - self.data_dir = data_dir + self.data_dir = local_operations_data self.create_dir() def view_description(self): @@ -1416,7 +1416,7 @@ def handle_work_locally_toggle(self): self.local_ftml_file = fs.path.combine( self.data_dir, fs.path.join( - "local_mscolab_data", self.user["username"], + "local_colabdata", self.user["username"], self.active_operation_name, "mscolab_operation.ftml"), ) self.ui.workingStatusLabel.setText( @@ -1444,7 +1444,7 @@ def handle_work_locally_toggle(self): def create_local_operation_file(self): with open_fs(self.data_dir) as mss_dir: - rel_file_path = fs.path.join('local_mscolab_data', self.user['username'], + rel_file_path = fs.path.join('local_colabdata', self.user['username'], self.active_operation_name, 'mscolab_operation.ftml') if mss_dir.exists(rel_file_path) is True: return diff --git a/mslib/msui/msui_mainwindow.py b/mslib/msui/msui_mainwindow.py index a32f5cdf4..0d725a75b 100644 --- a/mslib/msui/msui_mainwindow.py +++ b/mslib/msui/msui_mainwindow.py @@ -431,7 +431,16 @@ class MSUIMainWindow(QtWidgets.QMainWindow, ui.Ui_MSUIMainWindow): signal_permission_revoked = QtCore.pyqtSignal(int) signal_render_new_permission = QtCore.pyqtSignal(int, str) - def __init__(self, mscolab_data_dir=None, tutorial_mode=False, *args): + def __init__(self, local_operations_data=None, tutorial_mode=False, *args): + """ + This method initializes the main window of the application. + It sets up the user interface, icons, menu actions, and connects signals to slots. + + :param local_operations_data: Base path used by "work asynchronously" to store operations. + :param tutorial_mode: Whether to run the application in tutorial mode. Default is False. + :param args: Additional arguments to pass to the parent class. + + """ super().__init__(*args) self.tutorial_mode = tutorial_mode self.setupUi(self) @@ -507,7 +516,7 @@ def __init__(self, mscolab_data_dir=None, tutorial_mode=False, *args): self.statusBar.showMessage(self.status()) # Create MSColab instance to handle all MSColab functionalities - self.mscolab = mscolab.MSUIMscolab(parent=self, data_dir=mscolab_data_dir) + self.mscolab = mscolab.MSUIMscolab(parent=self, local_operations_data=local_operations_data) # Setting up MSColab Tab self.connectBtn.clicked.connect(self.mscolab.open_connect_window) diff --git a/tests/_test_mscolab/test_files.py b/tests/_test_mscolab/test_files.py index 35f0da5d2..10d3b8baa 100644 --- a/tests/_test_mscolab/test_files.py +++ b/tests/_test_mscolab/test_files.py @@ -64,7 +64,7 @@ def test_create_operation(self): # test for '/' in path assert self.fm.create_operation('test/path', 'sth', self.user) is False # check file existence - assert os.path.exists(os.path.join(mscolab_settings.MSCOLAB_DATA_DIR, 'test_path')) is True + assert os.path.exists(os.path.join(mscolab_settings.OPERATIONS_DATA, 'test_path')) is True # check creation in db p = Operation.query.filter_by(path="test_path").first() assert p is not None @@ -166,7 +166,7 @@ def test_modify_operation(self): assert self.fm.update_operation(op_id, 'path', 'dummy wrong', self.user) is False assert self.fm.update_operation(op_id, 'path', 'dummy/wrong', self.user) is False assert self.fm.update_operation(op_id, 'path', 'dummy', self.user) is True - assert os.path.exists(os.path.join(mscolab_settings.MSCOLAB_DATA_DIR, 'dummy')) + assert os.path.exists(os.path.join(mscolab_settings.OPERATIONS_DATA, 'dummy')) assert self.fm.update_operation(op_id, 'description', 'dummy', self.user) is True def test_delete_operation(self): diff --git a/tests/_test_mscolab/test_mscolab.py b/tests/_test_mscolab/test_mscolab.py index ca395d9f6..4bd93114c 100644 --- a/tests/_test_mscolab/test_mscolab.py +++ b/tests/_test_mscolab/test_mscolab.py @@ -73,20 +73,20 @@ def test_initial_state(self): def test_handle_db_reset(self): assert os.path.isdir(mscolab_settings.UPLOAD_FOLDER) - assert os.path.isdir(mscolab_settings.MSCOLAB_DATA_DIR) + assert os.path.isdir(mscolab_settings.OPERATIONS_DATA) all_operations = Operation.query.all() assert all_operations == [] operation_name = "Example" assert add_operation(operation_name, "Test Example") - assert os.path.isdir(os.path.join(mscolab_settings.MSCOLAB_DATA_DIR, operation_name)) + assert os.path.isdir(os.path.join(mscolab_settings.OPERATIONS_DATA, operation_name)) operation = Operation.query.filter_by(path=operation_name).first() assert operation.description == "Test Example" all_operations = Operation.query.all() assert len(all_operations) == 1 handle_db_reset() # check operation dir name removed - assert os.path.isdir(os.path.join(mscolab_settings.MSCOLAB_DATA_DIR, operation_name)) is False - assert os.path.isdir(mscolab_settings.MSCOLAB_DATA_DIR) + assert os.path.isdir(os.path.join(mscolab_settings.OPERATIONS_DATA, operation_name)) is False + assert os.path.isdir(mscolab_settings.OPERATIONS_DATA) assert os.path.isdir(mscolab_settings.UPLOAD_FOLDER) # query db for operation_name operation = Operation.query.filter_by(path=operation_name).first() diff --git a/tests/_test_mscolab/test_server.py b/tests/_test_mscolab/test_server.py index 33960915f..b80582e5b 100644 --- a/tests/_test_mscolab/test_server.py +++ b/tests/_test_mscolab/test_server.py @@ -50,7 +50,7 @@ def setup(self, mscolab_app, mscolab_managers): def test_initialized_managers(self, mscolab_managers): sockio, cm, fm = mscolab_managers - assert self.app.config['MSCOLAB_DATA_DIR'] == mscolab_settings.MSCOLAB_DATA_DIR + assert self.app.config['OPERATIONS_DATA'] == mscolab_settings.OPERATIONS_DATA assert 'Create a Flask-SocketIO server.' in sockio.__doc__ assert 'Class with handler functions for chat related functionalities' in cm.__doc__ assert 'Class with handler functions for file related functionalities' in fm.__doc__ @@ -442,7 +442,7 @@ def test_import_permissions(self): import_operation, token = self._create_operation(test_client, self.userdata, path="import") user = get_user(self.userdata[0]) another = get_user(another_user[0]) - fm = FileManager(self.app.config["MSCOLAB_DATA_DIR"]) + fm = FileManager(self.app.config["OPERATIONS_DATA"]) fm.add_bulk_permission(import_operation.id, user, [another.id], "viewer") current_operation, token = self._create_operation(test_client, self.userdata, path="current") response = test_client.post('/import_permissions', data={"token": token, diff --git a/tests/_test_mscolab/test_server_auth_required.py b/tests/_test_mscolab/test_server_auth_required.py index 14ab148e6..b9bbbc213 100644 --- a/tests/_test_mscolab/test_server_auth_required.py +++ b/tests/_test_mscolab/test_server_auth_required.py @@ -45,7 +45,7 @@ def setup(self, mscolab_app): def test_initialize_managers(self): app, sockio, cm, fm = initialize_managers(self.app) - assert app.config['MSCOLAB_DATA_DIR'] == mscolab_settings.MSCOLAB_DATA_DIR + assert app.config['OPERATIONS_DATA'] == mscolab_settings.OPERATIONS_DATA assert 'Create a Flask-SocketIO server.' in sockio.__doc__ assert 'Class with handler functions for chat related functionalities' in cm.__doc__ assert 'Class with handler functions for file related functionalities' in fm.__doc__ diff --git a/tests/_test_mscolab/test_utils.py b/tests/_test_mscolab/test_utils.py index 991b0ea60..2c8b1805e 100644 --- a/tests/_test_mscolab/test_utils.py +++ b/tests/_test_mscolab/test_utils.py @@ -83,7 +83,7 @@ def test_os_fs_create_dir(self): def test_create_file(self): create_files() # ToDo refactor to fs - assert os.path.exists(mscolab_settings.MSCOLAB_DATA_DIR) + assert os.path.exists(mscolab_settings.OPERATIONS_DATA) assert os.path.exists(mscolab_settings.UPLOAD_FOLDER) def _create_operation(self, test_client, userdata=None, path="firstflight", description="simple test"): diff --git a/tests/_test_msui/test_mscolab.py b/tests/_test_msui/test_mscolab.py index 010aed788..affc96de2 100644 --- a/tests/_test_msui/test_mscolab.py +++ b/tests/_test_msui/test_mscolab.py @@ -36,8 +36,8 @@ from PIL import Image +from tests.constants import ROOT_DIR import mslib.utils.auth -from mslib.mscolab.conf import mscolab_settings from mslib.mscolab.models import Permission, User from mslib.msui.flighttrack import WaypointsTableModel from PyQt5 import QtCore, QtTest, QtWidgets @@ -59,7 +59,7 @@ def setup(self, qtbot, mscolab_server): assert add_user_to_operation(path=self.operation_name, emailid=self.userdata[0]) self.user = get_user(self.userdata[0]) - self.main_window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR) + self.main_window = msui.MSUIMainWindow(local_operations_data=ROOT_DIR) self.main_window.create_new_flight_track() self.main_window.show() self.window = mscolab.MSColab_ConnectDialog(parent=self.main_window, mscolab=self.main_window.mscolab) @@ -281,7 +281,7 @@ def setup(self, qtbot, mscolab_app, mscolab_server): assert add_user(self.userdata3[0], self.userdata3[1], self.userdata3[2]) assert add_user_to_operation(path=self.operation_name3, access_level="collaborator", emailid=self.userdata3[0]) - self.window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR) + self.window = msui.MSUIMainWindow(local_operations_data=ROOT_DIR) self.window.create_new_flight_track() self.window.show() @@ -554,7 +554,7 @@ def assert_logout_text(): # ToDo verify all operations disabled again without a visual check @mock.patch("PyQt5.QtWidgets.QFileDialog.getSaveFileName", - return_value=(fs.path.join(mscolab_settings.MSCOLAB_DATA_DIR, 'test_export.ftml'), + return_value=(fs.path.join(ROOT_DIR, 'test_export.ftml'), "Flight track (*.ftml)")) def test_handle_export(self, mockbox, qtbot): self._connect_to_mscolab(qtbot) @@ -671,7 +671,7 @@ def test_handle_delete_operation(self, mocktext, qtbot): operation_name = "flight7" self._create_operation(qtbot, operation_name, "Description flight7") # check for operation dir is created on server - assert os.path.isdir(os.path.join(mscolab_settings.MSCOLAB_DATA_DIR, operation_name)) + assert os.path.isdir(os.path.join(ROOT_DIR, 'colabTestData', 'filedata', operation_name)) self._activate_operation_at_index(0) op_id = self.window.mscolab.get_recent_op_id() assert op_id is not None @@ -684,7 +684,7 @@ def test_handle_delete_operation(self, mocktext, qtbot): op_id = self.window.mscolab.get_recent_op_id() assert op_id is None # check operation dir name removed - assert os.path.isdir(os.path.join(mscolab_settings.MSCOLAB_DATA_DIR, operation_name)) is False + assert os.path.isdir(os.path.join(ROOT_DIR, operation_name)) is False @mock.patch("PyQt5.QtWidgets.QMessageBox.question", return_value=QtWidgets.QMessageBox.Yes) def test_handle_leave_operation(self, mockmessage, qtbot): diff --git a/tests/_test_msui/test_mscolab_admin_window.py b/tests/_test_msui/test_mscolab_admin_window.py index 515868e3e..961c65879 100644 --- a/tests/_test_msui/test_mscolab_admin_window.py +++ b/tests/_test_msui/test_mscolab_admin_window.py @@ -27,8 +27,8 @@ import mock import pytest -from mslib.mscolab.conf import mscolab_settings from PyQt5 import QtCore, QtTest, QtWidgets +from tests.constants import ROOT_DIR from mslib.msui import mscolab from mslib.msui import msui from mslib.mscolab.seed import add_user, get_user, add_operation, add_user_to_operation @@ -58,7 +58,7 @@ def setup(self, qtbot, mscolab_server): assert add_operation("tokyo", "test tokyo") assert add_user_to_operation(path="tokyo", emailid=self.userdata[0], access_level="creator") - self.window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR) + self.window = msui.MSUIMainWindow(local_operations_data=ROOT_DIR) self.window.create_new_flight_track() self.window.show() # connect and login to mscolab diff --git a/tests/_test_msui/test_mscolab_merge_waypoints.py b/tests/_test_msui/test_mscolab_merge_waypoints.py index 1d0fb9340..ebc95fdd3 100644 --- a/tests/_test_msui/test_mscolab_merge_waypoints.py +++ b/tests/_test_msui/test_mscolab_merge_waypoints.py @@ -29,8 +29,8 @@ import pytest import mslib.utils.auth +from tests.constants import ROOT_DIR from mslib.msui import flighttrack as ft -from mslib.mscolab.conf import mscolab_settings from PyQt5 import QtCore, QtTest from tests.utils import (mscolab_register_and_login, mscolab_create_operation, mscolab_delete_all_operations, mscolab_delete_user) @@ -44,7 +44,7 @@ class Test_Mscolab_Merge_Waypoints: def setup(self, qtbot, mscolab_app, mscolab_server): self.app = mscolab_app self.url = mscolab_server - self.window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR) + self.window = msui.MSUIMainWindow(local_operations_data=ROOT_DIR) self.window.create_new_flight_track() self.emailid = 'merge@alpha.org' yield @@ -53,7 +53,7 @@ def setup(self, qtbot, mscolab_app, mscolab_server): with self.app.app_context(): mscolab_delete_all_operations(self.app, self.url, self.emailid, 'abcdef', 'alpha') mscolab_delete_user(self.app, self.url, self.emailid, 'abcdef') - with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as mss_dir: + with fs.open_fs(ROOT_DIR) as mss_dir: if mss_dir.exists('local_mscolab_data'): mss_dir.removetree('local_mscolab_data') assert mss_dir.exists('local_mscolab_data') is False diff --git a/tests/_test_msui/test_mscolab_operation.py b/tests/_test_msui/test_mscolab_operation.py index 76e5ae334..76136a7a9 100644 --- a/tests/_test_msui/test_mscolab_operation.py +++ b/tests/_test_msui/test_mscolab_operation.py @@ -27,7 +27,7 @@ import pytest import datetime -from mslib.mscolab.conf import mscolab_settings +from tests.constants import ROOT_DIR from mslib.mscolab.models import Message, MessageType from PyQt5 import QtCore, QtTest, QtWidgets from mslib.msui import mscolab @@ -56,7 +56,7 @@ def setup(self, qtbot, mscolab_app, mscolab_server): assert add_operation(self.operation_name, "test europe") assert add_user_to_operation(path=self.operation_name, emailid=self.userdata[0]) self.user = get_user(self.userdata[0]) - self.window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR) + self.window = msui.MSUIMainWindow(local_operations_data=ROOT_DIR) self.window.create_new_flight_track() self.window.show() # connect and login to mscolab diff --git a/tests/_test_msui/test_mscolab_version_history.py b/tests/_test_msui/test_mscolab_version_history.py index 72f218c58..6e672e053 100644 --- a/tests/_test_msui/test_mscolab_version_history.py +++ b/tests/_test_msui/test_mscolab_version_history.py @@ -27,8 +27,8 @@ import pytest import mock -from mslib.mscolab.conf import mscolab_settings from PyQt5 import QtCore, QtTest, QtWidgets +from tests.constants import ROOT_DIR from mslib.msui import mscolab from mslib.msui import msui from mslib.mscolab.seed import add_user, get_user, add_operation, add_user_to_operation @@ -45,7 +45,7 @@ def setup(self, qtbot, mscolab_server): assert add_operation(self.operation_name, "test europe") assert add_user_to_operation(path=self.operation_name, emailid=self.userdata[0]) self.user = get_user(self.userdata[0]) - self.window = msui.MSUIMainWindow(mscolab_data_dir=mscolab_settings.MSCOLAB_DATA_DIR) + self.window = msui.MSUIMainWindow(local_operations_data=ROOT_DIR) self.window.create_new_flight_track() self.window.show() # connect and login to mscolab diff --git a/tests/fixtures.py b/tests/fixtures.py index 0a4e03f9c..5859c631b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -92,7 +92,7 @@ def mscolab_session_app(): """ _app = APP _app.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI - _app.config['MSCOLAB_DATA_DIR'] = mscolab_settings.MSCOLAB_DATA_DIR + _app.config['OPERATIONS_DATA'] = mscolab_settings.OPERATIONS_DATA _app.config['UPLOAD_FOLDER'] = mscolab_settings.UPLOAD_FOLDER return _app