Merge pull request #4 from CMSTrackerDPG/new_sso

.env_sample

@@ -0,0 +1,2 @@
+SSO_CLIENT_ID=
+SSO_CLIENT_SECRET=

.gitignore

@@ -203,4 +203,7 @@ dmypy.json
 ### Python Patch ###
 .venv/
 
-# End of https://www.gitignore.io/api/python,pycharm+all
+# End of https://www.gitignore.io/api/python,pycharm+all
+
+.env
+.vscode/

README.md

@@ -4,9 +4,10 @@
 
 # CERN Requests
 
-Enables using [requests]("https://github.com/requests/requests") without having to configure the CERN Root certificates.
 
-Inspired by [certifi](https://github.com/certifi/python-certifi), [requests-kerberos](https://github.com/requests/requests-kerberos) and [cern-sso-python](https://github.com/cerndb/cern-sso-python)
+Enables using [`requests`]("https://github.com/requests/requests") without having to configure the CERN Root certificates or getting an API access token manually.
+
+Inspired by [`certifi`](https://github.com/certifi/python-certifi), [`requests-kerberos`](https://github.com/requests/requests-kerberos), [`cern-sso-python`](https://github.com/cerndb/cern-sso-python) and [`api-access-examples`](https://gitlab.cern.ch/authzsvc/docs/api-access-examples/-/tree/master/python).
 
 The Root certificate bundle is copied from the [linuxsoft cern page](http://linuxsoft.cern.ch/cern/centos/7/cern/x86_64/repoview/CERN-CA-certs.html) and can also be created manually by downloading the CERN Grid Certification Authority files from [cafiles.cern.ch/cafiles](https://cafiles.cern.ch/cafiles/).
 
@@ -20,41 +21,82 @@ pip install cernrequests
 
 ## Prerequisites
 
-Request a [Grid User Certificate](https://ca.cern.ch/ca/) and convert into public and private key:
+### For sites requiring an SSL Grid certificate
+
+Request a [Grid User Certificate](https://ca.cern.ch/ca/) (with password) and convert into public and private key:
 
 ```bash
 mkdir -p ~/private
-openssl pkcs12 -clcerts -nokeys -in myCertificate.p12 -out ~/private/usercert.pem
-openssl pkcs12 -nocerts -in myCertificate.p12 -out ~/private/userkey.tmp.pem
-openssl rsa -in ~/private/userkey.tmp.pem -out ~/private/userkey.pem
+openssl pkcs12 -in myCertificate.p12 -clcerts -nokeys -out ~/private/usercert.pem  # Will ask for the certificate password
+openssl pkcs12 -in myCertificate.p12 -nocerts -nodes -out ~/private/userkey.pem  # Will ask for the certificate password
 ```
 
-The certificates have to be **passwordless**.
+The `.pem` certificates have to be **passwordless**.
+
+### For CERN APIs using the ""new"" SSO
+
+An `.env` file at the root of your project with the following variables set:
+- `SSO_CLIENT_ID`
+- `SSO_CLIENT_SECRET`
+
+To request them, you will need to register your application:
+
+1. Create an SSO registration for your application
+    on the [CERN Application Portal](https://application-portal.web.cern.ch):
+
+    ![](doc/create_registration_01.png)
+
+2. Add an application identifier and description:
+
+    ![](doc/create_registration_02.png)
+
+3. Edit the SSO application, go to the `SSO Registration` tab and click the plus button:
+
+    ![](doc/create_registration_03.png)
+
+4. Fill out the form of the new SSO registration as follows:
+
+    ![](doc/create_registration_04.png)
+
+    - You can put any value in the `Redirect URI(s)`, e.g. `http://localhost/*`
+    - Same for the `Base URL`
+    - Make sure you click `My application will need to get tokens using its own client ID and secret`.
+
+5. Submit the form:
+
+    ![](doc/create_registration_05.png)
+
+    Note the `client id` and `client secret` that the form will show you.
 
 ## Usage
 
 ### Example
 
+#### With Grid Certificates
 ```python
 import cernrequests
 
 url = "https://<your-cern-website>"
 response = cernrequests.get(url)
 ```
 
-### Cookies Example
+#### With API Token
 
-If you want to access a website which requires CERN Single Sign-on cookies you can do the following:
+If you want to access a website which requires a (""new"") CERN Single Sign-on token you can do the following:
 
 ```python
 import cernrequests
 
-url = "https://<your-cern-website>"
-cookies = cernrequests.get_sso_cookies(url)
-response = cernrequests.get(url, cookies=cookies)
+url = "https://<your-cern-website-url>"
+reponse = cernrequests.get_with_token(url, target_audience="<the SSO id of the target URL>")
 ```
+> **Note**
+> The `target_audience` depends on the SSO registration name of the _target_ application. E.g.
+> if you want to access the development instance of Run Registry, `target_audience` should be 
+> `dev-cmsrunregistry-sso-proxy`. 
+> In case of doubt, communicate with the app's developers directly. 
 
-### Alternative usage
+#### Alternative usage
 
 If you want to use ```requests``` directly without the CERN wrapper you can get the exact same functionality by doing:
 
@@ -71,6 +113,8 @@ response = requests.get(url, cert=cert, verify=ca_bundle)
 
 ## Configuration
 
+### Grid certificates
+
 The default user certificate paths are first ```~\private\``` and ```~\.globus\``` for fallback. The default *public* key file name is ```usercert.pem``` and the default *private* key file name is ```userkey.pem```
 
 You can configure the default grid user certificate path by setting the ```CERN_CERTIFICATE_PATH``` environment variable.
@@ -111,13 +155,23 @@ pytest
 
 ### I'm getting `certificate verify failed`! What should I do?
 
-The `cernrequests/cern-cacerts.pem` file has expired, and will need to be updated by the library maintainer. Download all the CA files from [here](https://ca.cern.ch/cafiles/certificates/Download.aspx?ca=cern) and convert them to `.pem` files, one-by-one by running:
+The `cernrequests/cern-cacerts.pem` file has expired, and will need to be updated by the library maintainer. 
 
-```bash
- openssl x509 -in <CERN certification authority file.crt> -out temp.pem -outform PEM
-```
+1. ```bash
+    git clone https://gitlab.cern.ch/linuxsupport/rpms/cern-ca-certs/
+    cd cern-ca-certs/src
+    make
+    ```
+    This will create a `CERN-bundle.pem` file.
+2. Rename it to `cern-cacerts.pem` and replace the original `.pem` certificate chain.
+
+Verify that the certs work by running `pytest`.
+
+
+### I'm getting `403 Client Error: Forbidden for url: https://login.cern.ch/adfs/ls/auth/sslclient` errors!1 What should I do?
 
-Then, merge the contents of each `.pem` file into a single `cern-cacerts.pem` file and replace the existing one. Verify that the certs work by running `pytest`.
+1. Your grid certificate may have expired. Try creating a new one.
+2. You may be trying to access a CERN webpage using a grid certificate, but this method may be deprecated. Make sure that the web page allows SSL certificate authentication.
 
 ## References
 
@@ -127,3 +181,4 @@ Then, merge the contents of each `.pem` file into a single `cern-cacerts.pem` fi
 - https://linux.web.cern.ch/linux/docs/cernssocookie.shtml
 - http://linuxsoft.cern.ch/cern/centos/7/cern/x86_64/repoview/CERN-CA-certs.html
 - https://ca.cern.ch/ca/
+- https://auth.docs.cern.ch

cernrequests/__init__.py

@@ -10,5 +10,5 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
-from .core import get
-from .cookies import get_sso_cookies
+from .core import get, get_with_token
+from .token import get_api_token

cernrequests/core.py

@@ -10,14 +10,51 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
+import os
 import requests
-
+from dotenv import load_dotenv
+from cernrequests.token import get_api_token
 from cernrequests.certs import default_user_certificate_paths, where
 
+load_dotenv()
+
 
 def get(url, params=None, **kwargs):
+    """
+    Method working with Grid Credentials.
+
+    Note: this is no longer supported by the "new"
+    CERN SSO (2023/06).
+
+    This method can still be used with CMSWEB.
+    """
     if "cert" not in kwargs:
         kwargs["cert"] = default_user_certificate_paths()
     if "verify" not in kwargs:
         kwargs["verify"] = where()
     return requests.get(url, params=params, **kwargs)
+
+
+def get_with_token(url, params=None, **kwargs):
+    """
+    Method working with the new (2023/06) SSO.
+    """
+    if "target_application" not in kwargs:
+        raise Exception("You must specify the target_application")
+    target_application = kwargs.pop("target_application")
+    client_id = os.environ.get("SSO_CLIENT_ID")
+    client_secret = os.environ.get("SSO_CLIENT_SECRET")
+    api_token, expiration_datetime = get_api_token(
+        client_id=client_id,
+        client_secret=client_secret,
+        target_application=target_application,
+    )
+    return requests.get(
+        url,
+        params=params,
+        headers={
+            "Authorization": "Bearer " + api_token,
+            "Content-Type": "application/json",
+        },
+        **kwargs
+    )

cernrequests/token.py

@@ -0,0 +1,54 @@
+"""
+API Token getter, based on
+https://gitlab.cern.ch/authzsvc/docs/api-access-examples/-/blob/master/python/api_token.py
+"""
+
+import logging
+import datetime
+import requests
+import jwt
+
+DEFAULT_SERVER = "auth.cern.ch"
+DEFAULT_REALM = "cern"
+DEFAULT_REALM_PREFIX = "auth/realms/{}"
+DEFAULT_TOKEN_ENDPOINT = "api-access/token"
+
+
+def get_token_endpoint(server=DEFAULT_SERVER, realm=DEFAULT_REALM):
+    """
+    Gets the token enpdoint path from the args
+    """
+    return "https://{}/{}/{}".format(
+        server, DEFAULT_REALM_PREFIX.format(realm), DEFAULT_TOKEN_ENDPOINT
+    )
+
+
+def get_api_token(
+    client_id, client_secret, target_application, token_endpoint=get_token_endpoint()
+):
+    logging.debug(
+        "[x] Getting API token as {} for {}".format(client_id, target_application)
+    )
+
+    r = requests.post(
+        token_endpoint,
+        auth=(client_id, client_secret),
+        data={"grant_type": "client_credentials", "audience": target_application},
+    )
+
+    if not r.ok:
+        msg = "ERROR getting token: {}".format(r.json())
+        logging.error(msg)
+        raise Exception(msg)
+
+    response_json = r.json()
+    token = response_json["access_token"]
+    expires_in_seconds = response_json["expires_in"]
+    expiration_datetime = datetime.datetime.now() + datetime.timedelta(
+        seconds=expires_in_seconds
+    )
+
+    # logging.debug(jwt.decode(token, verify=False))
+    logging.debug("[x] Token obtained")
+
+    return token, expiration_datetime

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name="cernrequests",
-    version="0.3.2",
+    version="0.4.0",
     desription="CERN wrapper around the requests package",
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -29,8 +29,8 @@
     author_email="peter.stein@cern.ch",
     packages=["cernrequests"],
     package_dir={"cernrequests": "cernrequests"},
-    package_data={"cernrequests": ["*.pem"]},
-    install_requires=["requests", "future"],
+    # package_data={"cernrequests": ["*.pem"]},
+    install_requires=["requests", "future", "python-dotenv", "pyjwt"],
     classifiers=[
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
     ],

tests/test_real_data.py

@@ -10,15 +10,14 @@
 # granted to it by virtue of its status as an Intergovernmental Organization
 # or submit itself to any jurisdiction.
 
+import os
 import json
-import pytest
-
 import requests
-from requests.exceptions import SSLError
 
 import cernrequests
 from cernrequests import certs
 from cernrequests.cookies import get_sso_cookies
+from cernrequests.core import get, get_with_token
 
 
 def test_dqmgui():
@@ -36,16 +35,19 @@ def test_rr():
     """
     RunRegistry requires cookies
     """
-    url = "https://cmsrunregistry.web.cern.ch/api/get_all_dataset_names_of_run/357756"
-    cert = certs.default_user_certificate_paths()
-    ca_bundle = certs.where()
-    cookies = get_sso_cookies(url, cert, verify=ca_bundle)
-    response = requests.get(url, cookies=cookies).json()
+
+    url = (
+        "https://dev-cmsrunregistry.web.cern.ch/api/get_all_dataset_names_of_run/357756"
+    )
+    response = get_with_token(
+        url, target_application="dev-cmsrunregistry-sso-proxy"
+    ).json()
     expected = [
-        "/Express/Collisions2022/DQM",
-        "/Express/Commissioning2022/DQM",
+        # TODO: Uncomment this when new SSO is deployed for production RR
+        # "/Express/Collisions2022/DQM",
+        # "/Express/Commissioning2022/DQM",
         "online",
-        "/PromptReco/Collisions2022/DQM",
+        # "/PromptReco/Collisions2022/DQM",
     ]
 
     assert expected == response