Merge pull request #93 from DUNE-DAQ/titavare/oracle-fixes

Made the database connections work, introduced pool recycling and sorted out the timezone issues by using utc

runnumber-rest/api.py

@@ -14,12 +14,13 @@
 ]
 
 import os
-from datetime import datetime
+import datetime as dt
 
 import flask
 from flask_restful import Api, Resource
 from flask_sqlalchemy import SQLAlchemy
-from sqlalchemy import func
+from sqlalchemy import func, event
+import re
 
 __all__ = ["app", "api", "db"]
 
@@ -32,6 +33,7 @@
     DEPLOYMENT_ENV=os.environ.get("DEPLOYMENT_ENV", "DEV"),
     RUN_START=int(os.getenv("RUN_START", "1000")),
     SQLALCHEMY_ECHO=False,
+        SQLALCHEMY_ENGINE_OPTIONS={"pool_pre_ping": True, "pool_recycle": 3600},
 )
 
 uri = app.config["SQLALCHEMY_DATABASE_URI"]
@@ -46,6 +48,14 @@
 PARSED_URI = urlparse(app.config["SQLALCHEMY_DATABASE_URI"])
 DB_TYPE = PARSED_URI.scheme
 
+@app.before_first_request
+def register_event_handlers():
+    @event.listens_for(db.engine, "handle_error")
+    def handle_exception(context):
+        if not context.is_disconnect and re.match(
+            r"^(?:DPI-1001|DPI-4011)", str(context.original_exception)
+        ):
+            context.is_disconnect = True
 
 # $ curl -u fooUsr:barPass -X GET np04-srv-021:30016//runnumber/get
 @api.resource("/runnumber/get")
@@ -114,7 +124,7 @@ def get(self, runNum):
             run = None
             with db.session.begin():
                 run = db.session.query(RunNumber).filter_by(rn=runNum).one()
-                run.stop_time = datetime.now()
+                run.stop_time = dt.datetime.utcnow()
             print(f"updateStopTimestamp: result {[run.start_time, run.stop_time]}")
             return flask.make_response(flask.jsonify([[[run.start_time, run.stop_time]]]))
         except Exception as err_obj:

runnumber-rest/database.py

@@ -13,6 +13,6 @@ class RunNumber(db.Model):
         'flag', db.Boolean, nullable=False, default=False
         )
     start_time = db.Column(
-        'start_time', db.TIMESTAMP(6), nullable=False, default=datetime.now
+        'start_time', db.TIMESTAMP(6), nullable=False, default=datetime.utcnow
     )
     stop_time = db.Column('stop_time', db.TIMESTAMP(6), nullable=True)

runregistry-rest/api.py

@@ -20,7 +20,8 @@
 from flask_caching import Cache
 from flask_restful import Api, Resource
 from flask_sqlalchemy import SQLAlchemy
-from sqlalchemy import desc, func
+from sqlalchemy import desc, event
+import re
 
 __all__ = ["app", "api", "db"]
 
@@ -37,13 +38,14 @@
     DEPLOYMENT_ENV=os.environ.get("DEPLOYMENT_ENV", "DEV"),
     RUN_START=int(os.getenv("RUN_START", "1000")),
     SQLALCHEMY_ECHO=False,
+        SQLALCHEMY_ENGINE_OPTIONS={"pool_pre_ping": True, "pool_recycle": 3600},
 )
 
 cache = Cache(app)
 db = SQLAlchemy(app)
 api = Api(app)
 
-import datetime
+import datetime as dt
 import urllib
 from urllib.parse import urlparse
 
@@ -53,6 +55,14 @@
 PARSED_URI = urlparse(app.config["SQLALCHEMY_DATABASE_URI"])
 DB_TYPE = PARSED_URI.scheme
 
+@app.before_first_request
+def register_event_handlers():
+    @event.listens_for(db.engine, "handle_error")
+    def handle_exception(context):
+        if not context.is_disconnect and re.match(
+            r"^(?:DPI-1001|DPI-4011)", str(context.original_exception)
+        ):
+            context.is_disconnect = True
 
 def cache_key():
     args = flask.request.args
@@ -93,7 +103,8 @@ def get(self, runNum):
             result = list(result)
             column_names = RunRegistryMeta.__table__.columns.keys()
             column_names.remove('filename') #Don't like this but only way to stay consistent with Oracle
-            return flask.make_response(flask.jsonify(column_names, [[*result]]))
+            cnu = [name.upper() for name in column_names]
+            return flask.make_response(flask.jsonify(cnu, [[*result]]))
         except Exception as err_obj:
             print(f"Exception:{err_obj}")
             return flask.make_response(flask.jsonify({"Exception": f"{err_obj}"}))
@@ -127,7 +138,8 @@ def get(self, amount):
             result = [list(row) for row in result]
             column_names = RunRegistryMeta.__table__.columns.keys()
             column_names.remove('filename') #Don't like this but only way to stay consistent with Oracle
-            return flask.make_response(flask.jsonify(column_names, [*result]))
+            cnu = [name.upper() for name in column_names]
+            return flask.make_response(flask.jsonify(cnu, [*result]))
         except Exception as err_obj:
             return flask.make_response(flask.jsonify({"Exception": f"{err_obj}"}))
 
@@ -167,7 +179,7 @@ def get(self, runNum):
             print(f"Exception:{err_obj}")
             return flask.make_response(flask.jsonify({"Exception": f"{err_obj}"})) 
 
-# $ curl -u fooUsr:barPass -F "run_number=1000" -F "det_id=foo" -F "run_type=bar" -F "software_version=dunedaq-vX.Y.Z" -F "file=@sspconf.tar.gz" -X POST np04-srv-017:30015/runregistry/insertRun/
+# $ curl -u fooUsr:barPass -F "run_num=1000" -F "det_id=foo" -F "run_type=bar" -F "software_version=dunedaq-vX.Y.Z" -F "file=@sspconf.tar.gz" -X POST np04-srv-017:30015/runregistry/insertRun/
 @api.resource("/runregistry/insertRun/")
 class insertRun(Resource):
     """
@@ -181,7 +193,7 @@ def post(self):
         local_file_name = None
         try:
             # Ensure form fields
-            run_number = flask.request.form.get("run_number")
+            run_number = flask.request.form.get("run_num")
             det_id = flask.request.form.get("det_id")
             run_type = flask.request.form.get("run_type")
             software_version = flask.request.form.get("software_version")
@@ -233,8 +245,8 @@ def post(self):
                 os.remove(local_file_name)
 
 
-# $ curl -u fooUsr:barPass -X GET np04-srv-017:30015/runregistry/updatestop/<int:runNum>
-@api.resource("/runregistry/updatestop/<int:runNum>")
+# $ curl -u fooUsr:barPass -X GET np04-srv-017:30015/runregistry/updateStopTime/<int:runNum>
+@api.resource("/runregistry/updateStopTime/<int:runNum>")
 class updateStopTimestamp(Resource):
     """
     set and record the stop time for the run into the database
@@ -248,7 +260,7 @@ def get(self, runNum):
             run = None
             with db.session.begin():
                 run = db.session.query(RunRegistryMeta).filter_by(run_number=runNum).one()
-                run.stop_time = datetime.now()
+                run.stop_time = dt.datetime.utcnow()
             print(f"updateStopTimestamp: result {[run.start_time, run.stop_time]}")
             return flask.make_response(flask.jsonify([[[run.start_time, run.stop_time]]]))
         except Exception as err_obj:

runregistry-rest/database.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+import datetime as dt
 
 from api import db
 
@@ -10,7 +10,7 @@ class RunRegistryMeta(db.Model):
         'run_number', db.Integer, primary_key=True, autoincrement=True, nullable=False
     )
     start_time = db.Column(
-        'start_time', db.TIMESTAMP(6), nullable=False, default=datetime.now
+        'start_time', db.TIMESTAMP(6), nullable=False, default=dt.datetime.utcnow
     )
     stop_time = db.Column(
         'stop_time', db.TIMESTAMP(6), nullable=True
@@ -27,10 +27,11 @@ class RunRegistryMeta(db.Model):
     software_version = db.Column(
         'software_version', db.String(40)
     )
+    configs = db.relationship('RunRegistryConfigs', backref='meta')
 
 class RunRegistryConfigs(db.Model):
     run_number = db.Column(
-        'run_number', db.Integer,  primary_key=True, autoincrement=True, nullable=False
+        'run_number', db.Integer, db.ForeignKey(RunRegistryMeta.run_number), primary_key=True, nullable=False
     )
     configuration = db.Column(
         'configuration', db.LargeBinary, nullable=False