Read cron_interval_minutes from env var if not defined in the config …

…yaml.

README.md

@@ -27,7 +27,7 @@ Then fill in the relevant information in the **config.yml** file in the root fol
 
  1. Update the service parameters 
 	 - **run_as_service**:  True or False. True will run this application as a service with an interval cron service on a schedule specified in *service_schedule* parameter. False is useful to setup this application up to be invoked by crontab. The crontab frequency should then be specified using the *cron_interval_minutes* parameter.
-	 - **cron_interval_minutes**:  Required only if *run_as_service* is False. 
+	 - **cron_interval_minutes**:  Required only if *run_as_service* is False. If not defined, it will try to read the `CRON_INTERVAL_MINUTES` environment variable.
 	 - **service_schedule**: Required only if *run_as_service* is True. The *hour* attribute specifies all the hours (0 - 23, comma separated) to invoke the application to fetch logs. Use * as wildcard to invoke the application every hour. The *minute* attribute specifies all the minutes (0 - 59, comma separated) at which invoke the application to fetch logs. For example { "hour" = "*", "minute" = "0, 15, 30, 45"}  will fetch logs every hour on the hour as well as the 15 minute, 30 minute and 45th minute past every hour. 
 
  2. Update the **instances** section with the *name*, connection  

config.yml.sample

@@ -39,9 +39,9 @@ queries: [
     api_ver: "58.0"
   },
   {
-    query: "SELECT EventName, EventType, UsageType, Client, Value, StartDate, EndDate FROM PlatformEventUsageMetric WHERE TimeSegment='FifteenMinutes' AND StartDate >= {start_date} AND EndDate <= {now}",
+    query: "SELECT EventName, EventType, UsageType, Client, Value, StartDate, EndDate FROM PlatformEventUsageMetric WHERE TimeSegment='FifteenMinutes' AND StartDate >= {start_date} AND EndDate <= {end_date}",
     env: {
-      now: "now()",
+      end_date: "now()",
       start_date: "now(timedelta(minutes=-60))"
     },
     api_ver: "58.0",

src/__main__.py

@@ -8,7 +8,7 @@
 from apscheduler.schedulers.background import BlockingScheduler
 from pytz import utc
 from yaml import Loader, load
-
+from newrelic_logging.env import get_var, var_exists
 from newrelic_logging.integration import Integration
 
 config_dir = None
@@ -73,7 +73,13 @@ def main():
     run_as_service = config.get('run_as_service', False)
 
     if not run_as_service:
-        cron_interval = config.get('cron_interval_minutes', 60)
+        if 'cron_interval_minutes' in config:
+            cron_interval = config['cron_interval_minutes']
+        elif var_exists("CRON_INTERVAL_MINUTES"):
+            cron_interval = int(get_var("CRON_INTERVAL_MINUTES"))
+        else:
+            cron_interval = 60
+
         integration = Integration(config, event_mapping, cron_interval)
         integration.run()
     else:

src/newrelic_logging/auth_env.py → src/newrelic_logging/env.py

@@ -1,5 +1,15 @@
 import os
 
+def var_exists(var_name):
+    return var_name in os.environ
+
+def get_var(var_name, default = None):
+    if default == None:
+        # Can raise exception
+        return os.environ[var_name]
+    else:
+        return os.environ.get(var_name, default)
+
 class AuthEnv:
     SF_GRANT_TYPE = 'SF_GRANT_TYPE'
     SF_CLIENT_ID =  'SF_CLIENT_ID'
@@ -23,41 +33,34 @@ def __init__(self, prefix):
         self.SF_AUDIENCE = prefix + self.SF_AUDIENCE
 
     def get_grant_type(self, default=None):
-        return self.get(self.SF_GRANT_TYPE, default)
+        return get_var(self.SF_GRANT_TYPE, default)
 
     def get_client_id(self, default=None):
-        return self.get(self.SF_CLIENT_ID, default)
+        return get_var(self.SF_CLIENT_ID, default)
 
     def get_client_secret(self, default=None):
-        return self.get(self.SF_CLIENT_SECRET, default)
+        return get_var(self.SF_CLIENT_SECRET, default)
 
     def get_username(self, default=None):
-        return self.get(self.SF_USERNAME, default)
+        return get_var(self.SF_USERNAME, default)
 
     def get_password(self, default=None):
-        return self.get(self.SF_PASSWORD, default)
+        return get_var(self.SF_PASSWORD, default)
 
     def get_private_key(self, default=None):
-        return self.get(self.SF_PRIVATE_KEY, default)
+        return get_var(self.SF_PRIVATE_KEY, default)
 
     def get_subject(self, default=None):
-        return self.get(self.SF_SUBJECT, default)
+        return get_var(self.SF_SUBJECT, default)
 
     def get_audience(self, default=None):
-        return self.get(self.SF_AUDIENCE, default)
+        return get_var(self.SF_AUDIENCE, default)
 
     def get_license_key(self, default=None):
-        return self.get(self.NR_LICENSE_KEY, default)
+        return get_var(self.NR_LICENSE_KEY, default)
 
     def get_account_id(self, default=None):
-        return self.get(self.NR_ACCOUNT_ID, default)
-
-    def get(self, var_name, default):
-        if default == None:
-            # Can raise exception
-            return os.environ[var_name]
-        else:
-            return os.environ.get(var_name, default)
+        return get_var(self.NR_ACCOUNT_ID, default)
 
 class Auth:
     access_token = None

src/newrelic_logging/integration.py

@@ -2,14 +2,18 @@
 from .http_session import new_retry_session
 from .newrelic import NewRelic
 from .salesforce import SalesForce, SalesforceApiException, DataCache
-from .auth_env import AuthEnv
+from .env import AuthEnv
 from enum import Enum
 from .telemetry import Telemetry
 
 class DataFormat(Enum):
     LOGS = 1
     EVENTS = 2
 
+#TODO: get general api_ver that applies to all queries of one instance
+#TODO: move queries to the instance level, so we can have different queries for each instance.
+#TODO: also keep general queries that apply to all instances.
+
 class Integration:
     numeric_fields_list = set()
 
@@ -20,11 +24,8 @@ def __init__(self, config, event_type_fields_mapping, initial_delay):
             instance_name = instance['name']
             labels = instance['labels']
             labels['nr-labs'] = 'data'
-            #TODO: get general api_ver that applies to all queries of one instance
             prefix = instance['arguments'].get('auth_env_prefix', '')
             auth_env = AuthEnv(prefix)
-            #TODO: move queries to the instance level, so we can have different queries for each instance.
-            #TODO: also keep general queries that apply to all instances.
             if 'queries' in config:
                 client = SalesForce(auth_env, instance_name, instance['arguments'], event_type_fields_mapping, initial_delay, config['queries'])
             else:

src/newrelic_logging/query.py

@@ -19,4 +19,37 @@ def set_query(self, query: str) -> None:
 
     def get_env(self) -> dict:
         return self.env
-
+
+# NOTE: this sandbox can be jailbroken using the trick to exec statements inside an exec block, and run an import (and other tricks):
+# https://book.hacktricks.xyz/generic-methodologies-and-resources/python/bypass-python-sandboxes#operators-and-short-tricks
+# https://stackoverflow.com/a/3068475/2076108
+# Would be better to use a real sandbox like https://pypi.org/project/RestrictedPython/ or https://doc.pypy.org/en/latest/sandbox.html
+# or parse a small language that only supports funcion calls and binary expressions.
+def sandbox(code):
+    __import__ = None
+    __loader__ = None
+    __build_class__ = None
+    exec = None
+
+    from datetime import datetime, timedelta
+
+    def sf_time(t: datetime):
+        return t.isoformat(timespec='milliseconds') + "Z"
+
+    def now(delta: timedelta = None):
+        if delta:
+            return sf_time(datetime.utcnow() + delta)
+        else:
+            return sf_time(datetime.utcnow())
+
+    try:
+        return eval(code)
+    except Exception as e:
+        return e
+
+def substitute(args: dict, query_template: str, env: dict) -> str:
+    for key, command in env.items():
+        args[key] = sandbox(command)
+    for key, val in args.items():
+        query_template = query_template.replace('{' + key + '}', val)
+    return query_template

src/newrelic_logging/salesforce.py

@@ -10,9 +10,8 @@
 from requests import RequestException
 import copy
 import hashlib
-from .query_env import substitute
-from .auth_env import Auth
-from .query import Query
+from .env import Auth
+from .query import Query, substitute
 from .telemetry import Telemetry
 
 class LoginException(Exception):