Merge branch 'master' into training

.env_sample

@@ -0,0 +1,25 @@
+DJANGO_DATABASE_ENGINE=django.db.backends.postgresql
+DJANGO_DEBUG=False
+DJANGO_DATABASE_HOST=127.0.0.1
+DJANGO_DATABASE_NAME=certifier
+DJANGO_DATABASE_USER=
+DJANGO_DATABASE_PASSWORD=
+DJANGO_DATABASE_PORT=5432
+DJANGO_SECRET_KEY=
+CSRF_TRUSTED_ORIGINS=
+
+
+# For connecting to vocms virtual machines
+DJANGO_SECRET_ACC=
+DJANGO_SECRET_PASS=
+
+# For connecting to OMS
+OMS_CLIENT_ID=
+OMS_CLIENT_SECRET=
+# OMS_API_URL=https://cmsoms.cern.ch/agg/api
+# OMS_API_AUDIENCE=cmsoms-prod
+
+# For connecting to Run Registry. You can reuse 
+# the OMS_CLIENT_ID and OMS_CLIENT_SECRET
+SSO_CLIENT_ID=
+SSO_CLIENT_SECRET=

.github/workflows/django.yml

@@ -41,7 +41,11 @@ jobs:
         git config --global url."https://${{ secrets.CERN_GITLAB_USER }}:${{ secrets.CERN_GITLAB_TOKEN }}@gitlab.cern.ch".insteadOf https://gitlab.cern.ch       
         git config --global url."https://${{ secrets.CERN_GITLAB_USER }}:${{ secrets.CERN_GITLAB_TOKEN }}@gitlab.cern.ch".insteadOf https://gitlab.cern.ch
         python -m pip install --upgrade pip
+        
         pip install -r requirements.txt
+
+        pip install --index-url https://test.pypi.org/simple runregistry==1.0.0
+
         pip install --upgrade pytest pytest-django pytest-cov codecov mixer selenium
 
     - name: Run Unit Tests
@@ -56,7 +60,8 @@ jobs:
         DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }}
         OMS_CLIENT_ID: ${{ secrets.OMS_CLIENT_ID }}
         OMS_CLIENT_SECRET: ${{ secrets.OMS_CLIENT_SECRET }}
-
+        CERN_SSO_REGISTRATION_CLIENT_ID: ${{ secrets.CERN_SSO_REGISTRATION_CLIENT_ID }}
+        CERN_SSO_REGISTRATION_CLIENT_SECRET: ${{ secrets.CERN_SSO_REGISTRATION_CLIENT_SECRET }}
       run: |
         PYTHONWARNINGS=all pytest --ds=dqmhelper.test_ci_settings --cov=./ --ignore addrefrun/tests/test_addrefrun_views.py --ignore certifier/tests/test_certifier_views.py --ignore oms/tests/test_oms_utils.py --ignore oms/tests/test_oms_models.py
         codecov

.gitignore

@@ -274,5 +274,13 @@ wsgi/static/
 
 # End of https://www.gitignore.io/api/django,python,pycharm+all
 
+# Temp files when creating presentations
 *.odp
 
+# Directory where static files go after collectstatic
+sock/
+
+# Redis db files
+*.rdb
+
+.env_*

.s2i/bin/assemble

@@ -1,6 +1,6 @@
 #!/bin/bash
 echo "Before assembling"
-git config --global url."https://$CERN_GITLAB_USER:$CERN_GITLAB_TOKEN@gitlab.cern.ch".insteadOf https://gitlab.cern.ch    
+git config --global url."https://$CERN_GITLAB_USER:$CERN_GITLAB_TOKEN@gitlab.cern.ch".insteadOf https://gitlab.cern.ch   
 /usr/libexec/s2i/assemble
 rc=$?
 

certifier/templates/certifier/certify.html

@@ -78,7 +78,19 @@ <h1>
             <div class="col-md-4">
                 <div class="card shadow-sm bg-white">
                     <div class="card-body">
-                        <h6 class="card-title text-primary">Type</h6>
+                        <div class="row">
+                            <div class="col-11">
+                                <h6 class="card-title text-primary">Type</h6>
+                            </div>
+                            <div class="col-1">
+                                {% comment %} Allow editing of OMS attributes {% endcomment %}
+                                {% if form.external_info_complete.value %}
+                                <div class="row justify-content-center">
+                                    <a class="btn btn-light" title="Edit OMS info" href="./?external_info_complete=false"><i class="bi bi-pencil-fill pl-1"></i></a>
+                                </div>
+                                {% endif %}
+                            </div>
+                        </div>
                         <div class="row m-1">
                             <div class="col-md-4 text-right">
                                 <strong>Run type</strong>
@@ -155,8 +167,19 @@ <h6 class="card-title text-primary">Type</h6>
             <div class="col-md-4">
                 <div class="card shadow-sm">
                     <div class="card-body">
-
-                        <h6 class="card-title text-success">Lumi</h6>
+                        <div class="row">
+                            <div class="col-11">
+                                <h6 class="card-title text-success">Lumi</h6>
+                            </div>
+                            <div class="col-1">
+                                {% comment %} Allow editing of OMS attributes {% endcomment %}
+                                {% if form.external_info_complete.value %}
+                                <div class="row justify-content-center">
+                                    <a class="btn btn-light" title="Edit OMS info" href="./?external_info_complete=false"><i class="bi bi-pencil-fill pl-1"></i></a>
+                                </div>
+                                {% endif %}
+                            </div>
+                        </div>
                         <div class="row m-1">
                             <div class="col-md-5 text-right">
                                 <strong>Lumisections</strong>
@@ -237,7 +260,19 @@ <h6 class="card-title text-success">Lumi</h6>
             <div class="col-md-4">
                 <div class="card shadow-sm">
                     <div class="card-body">
-                        <h6 class="card-title text-info">Fill</h6>
+                        <div class="row">
+                            <div class="col-11">
+                                <h6 class="card-title text-info">Fill</h6>
+                            </div>
+                            <div class="col-1">
+                                {% comment %} Allow editing of OMS attributes {% endcomment %}
+                                {% if form.external_info_complete.value %}
+                                <div class="row justify-content-center">
+                                    <a class="btn btn-light" title="Edit OMS info" href="./?external_info_complete=false"><i class="bi bi-pencil-fill pl-1"></i></a>
+                                </div>
+                                {% endif %}
+                            </div>
+                        </div>
                         <div class="row m-1">
                             <div class="col-md-5 text-right">
                                 <strong>Fill number</strong>

certifier/tests/conftest.py

@@ -552,7 +552,7 @@ def runs_for_slr():
 @pytest.fixture
 def shifter(django_user_model):
     user = mixer.blend(get_user_model(), username=SHIFTER1_USERNAME, password=PASSWORD)
-    user.extra_data = {"groups": ["tkdqmdoctor-shifters"]}
+    user.extra_data = {"cern_roles": ["shifter"]}
     user.update_privilege()
     user.save()
     return user

certifier/views.py

@@ -24,6 +24,7 @@
 from oms.exceptions import (
     OmsApiFillNumberNotFound,
     OmsApiRunNumberNotFound,
+    OmsApiDataInvalidError,
     RunRegistryNoAvailableDatasets,
     RunRegistryReconstructionNotFound,
 )
@@ -204,6 +205,7 @@ def dispatch(self, request, *args, **kwargs):
             ParseError,
             OmsApiRunNumberNotFound,
             OmsApiFillNumberNotFound,
+            OmsApiDataInvalidError,
         ) as e:
             # If OMS API does not contain the info required,
             # or OMS is unreachable, create the run with minimal
@@ -222,6 +224,10 @@ def get(self, request, run_number: int, reco: str = None):
         logger.debug(
             f"Requesting certification of run {run_number} {reco if reco else ''}"
         )
+        # Allow GUI user to force update of the OMS data
+        # A bit of a machete code, in order to
+        if request.GET.get("external_info_complete", None) == "false":
+            self.external_info_complete = False
 
         context = {
             "run_number": run_number,

dqmhelper/settings.py

@@ -10,17 +10,15 @@
 https://docs.djangoproject.com/en/2.1/ref/settings/
 """
 
-import os
 from pathlib import Path
-
 from decouple import config
 from django.contrib.messages import constants as messages
 
 MESSAGE_TAGS = {
     messages.ERROR: "danger",
 }
 # Version to display in order to keep track of changes
-CERTHELPER_VERSION = "1.10.2"
+CERTHELPER_VERSION = "1.11.1"
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -48,6 +46,7 @@
 CSRF_TRUSTED_ORIGINS = [config("CSRF_TRUSTED_ORIGINS", default="")]
 
 INSTALLED_APPS = [
+    "daphne",
     "channels",
     "remotescripts",
     "openruns.apps.OpenrunsConfig",
@@ -75,8 +74,7 @@
     "allauth",
     "allauth.account",
     "allauth.socialaccount",
-    "allauth.socialaccount.providers.cern",
-    "allauth.socialaccount.providers.github",
+    "allauth.socialaccount.providers.openid_connect",
     "widget_tweaks",
     "django_extensions",
     "django_tables2",
@@ -175,7 +173,11 @@
 
 USE_TZ = True
 
-SITE_ID = 2
+# This must be added from the admin page. Then, you
+# can see the id by:
+# from django.contrib.sites.models import Site
+# Site.objects.all()
+SITE_ID = config("SITE_ID", default=2, cast=int)
 
 LOGGING = {
     "version": 1,
@@ -221,3 +223,36 @@
 
 # When Upgraded to Django 3.2 - RELEASE 06.04.2021
 DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+ACCOUNT_EMAIL_VERIFICATION = "none"
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+
+SOCIALACCOUNT_PROVIDERS = {
+    "openid_connect": {
+        "SERVERS": [
+            {
+                "id": "cern",  # 30 characters or less
+                "name": "CERN",
+                "server_url": "https://auth.cern.ch/auth/realms/cern",
+                # Optional token endpoint authentication method.
+                # May be one of "client_secret_basic", "client_secret_post"
+                # If omitted, a method from the the server's
+                # token auth methods list is used
+                "token_auth_method": "client_secret_post",
+                "APP": {
+                    "client_id": config("CERN_SSO_REGISTRATION_CLIENT_ID"),
+                    "secret": config("CERN_SSO_REGISTRATION_CLIENT_SECRET"),
+                },
+            },
+        ]
+    }
+}
+
+# This is used to get the public key and decode access tokens
+# for users when they login. The URL can be found under the
+# jwks_uri key of the JSON pointed to by the server_url of
+# CERN's well-known config URL:
+# https://auth.cern.ch/auth/realms/cern/.well-known/openid-configuration
+CERN_SSO_JWKS_URI = (
+    "https://auth.cern.ch/auth/realms/cern/protocol/openid-connect/certs"
+)

dqmhelper/test_settings.py

@@ -1,3 +1,4 @@
+import os
 from .settings import *
 
 if os.environ.get("GITHUB_WORKFLOW"):

listruns/tests/test_listruns_views.py

@@ -121,7 +121,7 @@ def test_update_not_logged_dispatch(self):
         req = RequestFactory().get(reverse("listruns:update", kwargs=arguments))
 
         user = mixer.blend(get_user_model())
-        user.extra_data = {"groups": ["tkdqmdoctor-shiftleaders"]}
+        user.extra_data = {"cern_roles": ["shiftleader"]}
         user.update_privilege()
         user.save()
 
@@ -158,7 +158,7 @@ def test_update_get_success_url(self):
         req = RequestFactory().get(reverse("listruns:update", kwargs=arguments))
 
         user = mixer.blend(get_user_model())
-        user.extra_data = {"groups": ["tkdqmdoctor-shiftleaders"]}
+        user.extra_data = {"cern_roles": ["shiftleader"]}
         user.update_privilege()
         user.save()
 
@@ -195,7 +195,7 @@ def test_update_same_user_or_shiftleader_true(self):
         req = RequestFactory().get(reverse("listruns:update", kwargs=arguments))
 
         user = mixer.blend(get_user_model())
-        user.extra_data = {"groups": ["tkdqmdoctor-shiftleaders"]}
+        user.extra_data = {"cern_roles": ["shiftleader"]}
         user.update_privilege()
         user.save()
 
@@ -233,7 +233,7 @@ def test_update_same_user_or_shiftleader_false(self):
 
         user = mixer.blend(get_user_model())
         user_aux = mixer.blend(get_user_model())
-        user.extra_data = {"groups": ["tkdqmdoctor-shiftleaders"]}
+        user.extra_data = {"cern_roles": ["shiftleader"]}
         user.update_privilege()
         user.save()
 

oms/exceptions.py

@@ -10,6 +10,13 @@ class OmsApiRunNumberNotFound(Exception):
     """
 
 
+class OmsApiDataInvalidError(Exception):
+    """
+    Raised when data from OMS violates a constraint in our DB,
+    e.g. Colliding bunches < 0
+    """
+
+
 class RunRegistryNoAvailableDatasets(Exception):
     """
     Raised when no datasets were found in RunRegistry

oms/utils.py

@@ -6,6 +6,7 @@
 from oms.exceptions import (
     OmsApiFillNumberNotFound,
     OmsApiRunNumberNotFound,
+    OmsApiDataInvalidError,
     RunRegistryNoAvailableDatasets,
     RunRegistryReconstructionNotFound,
 )
@@ -161,7 +162,20 @@ def oms_retrieve_fill(fill_number: int) -> OmsFill:  # pragma: no cover
     try:
         with transaction.atomic():
             OmsFill.objects.create(**fill_kwargs)
-    except IntegrityError:
+    except IntegrityError as e:
+        logger.error(f"Failed to create OmsFill object in DB: {repr(e)}")
+        if "violates check constraint" in repr(e):
+            raise OmsApiDataInvalidError(repr(e))
+        # This try-except block was probably added for the case where more than one users simultaneously
+        # ask to certify the same run, meaning that only one call of this function
+        # will actually manage to create the entry, all others throwing an Integrity Error.
+        # Not raising an exception here, though, allows ALL kinds of IntegrityErrors to pass through,
+        # including ones that have to do with the actual data from OMS being bad.
+        # An example was an incident where the bunches_colliding that OMS was returning was negative,
+        # raising an IntegrityError here, which failed to create the OmsFill entry, hence
+        # raising an exception on the return statement without a descriptive message.
+        # For this reason, it's better to just crash early, with a meaningful reason.
+        # There's, of course, room for improvement here.
         OmsFill.objects.filter(fill_number=fill_number).update(**fill_kwargs)
 
     return OmsFill.objects.get(fill_number=fill_number)
@@ -250,10 +264,12 @@ def oms_retrieve_run(run_number: int) -> OmsRun:  # pragma: no cover
     run_kwargs["lumisections"] = get_oms_lumisection_count(run_number)
 
     try:
+        # Make sure that, if more than one user requests a run that had not been added to the DB,
+        # it will only be added once.
         with transaction.atomic():
             OmsRun.objects.create(fill=fill, **run_kwargs)
     except IntegrityError as e:
-        logger.warning(f"{e} trying to create OmsRun")
+        logger.error(f"Failed to create OmsRun object: {repr(e)}")
         OmsRun.objects.filter(run_number=run_number).update(**run_kwargs)
 
     return OmsRun.objects.get(run_number=run_number)

omsapi/apiconfig.py

@@ -1,7 +1,9 @@
 from decouple import config
 
-API_URL = 'https://vocms0185.cern.ch/agg/api'
-API_VERSION = 'v1'
-API_AUDIENCE = 'cmsoms-int-0185'
-OMS_CLIENT_ID = config('OMS_CLIENT_ID')
-OMS_CLIENT_SECRET = config('OMS_CLIENT_SECRET')
+# See other options here:
+# https://gitlab.cern.ch/cmsoms/oms-api-client/-/wikis/uploads/01fe5b10560e76849ce636cf53e59e20/OMS_CERN_OpenID_API__2022_.pdf
+API_URL = config("OMS_API_URL", "https://cmsoms.cern.ch/agg/api")
+API_VERSION = "v1"
+API_AUDIENCE = config("OMS_API_AUDIENCE", "cmsoms-prod")
+OMS_CLIENT_ID = config("OMS_CLIENT_ID")
+OMS_CLIENT_SECRET = config("OMS_CLIENT_SECRET")

openshift-start-up-script.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
-
 python manage.py collectstatic --noinput
 python manage.py migrate --run-syncdb
 python manage.py clear_scripts_running_status

pytest.ini

@@ -7,6 +7,5 @@ filterwarnings =
     ignore::DeprecationWarning
     ignore::PendingDeprecationWarning
     ignore::ImportWarning
-    ignore::numba.errors.NumbaWarning
     ignore::UserWarning
 	ignore::urllib3.exceptions.InsecureRequestWarning