Load query config from multiple files.

README.md

@@ -201,6 +201,20 @@ queries: [
 
 In this case, the integration will combine the fields `Client`, `Value`, `StartDate`, and `EndDate` to form a unique identifier for each event of the type `PlatformEventUsageMetric`.
 
+The `queries` key can also contain references to other YAML files. Like so:
+
+```yaml
+queries: [
+    "my_file_1.yml",
+    "my_file_2.yml",
+    {
+        query: "...",
+    }
+]
+```
+
+These files must contain a `queries` key with the same format already explained. With the limitation that these sub-config files can't contain references to other files, only query objects.
+
 ### Query env
 
 It's possible to define a set of environment variables for a specific query, we do it by setting the key `env`:
@@ -242,7 +256,7 @@ Fields in the CSV come as strings, but some of them are actually numeric and we
 
 ## Telemetry
 
-This integration automatically generates New Relic logs to trace its health state and correct functioning. It generates two kinds of logs: Info and Error. Info are things like, "Logs processed" or "Correctly authenticated". Errors are things like "Failed posting logs to New Relic" or "SOQL query failed".
+This integration automatically generates New Relic logs to trace its health state and correct functioning. It generates two diffeerent levels of logs: info and error. Info are things like, "Logs processed" or "Correctly authenticated". Errors are things like "Failed posting logs to New Relic" or "SOQL query failed".
 
 ## Usage
 

config_sample.yml

@@ -47,7 +47,8 @@ queries: [
     },
     api_ver: "58.0",
     timestamp_attr: StartDate,
-  }
+  },
+  "other_queries.yml"
 ]
 newrelic:
   data_format: "events"

src/__main__.py

@@ -37,8 +37,7 @@
 numeric_fields_file = f'{config_dir}/numeric_fields.yml'
 
 def main():
-    with open(config_file) as stream:
-        config = load(stream, Loader=Loader)
+    config = load_config(config_file)
 
     if not os.path.exists(event_mapping_file):
         print(f'event_mapping_file {event_mapping_file} not found, so event mapping will not be used')
@@ -103,5 +102,26 @@ def main():
         print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
         scheduler.start()
 
+def load_config(config_file: str):
+    with open(config_file) as stream:
+        config = load(stream, Loader=Loader)
+    new_queries = []
+    if 'queries' in config:
+        for query in config['queries']:
+            if type(query) is str:
+                with open(query) as stream:
+                    sub_query_config = load(stream, Loader=Loader)
+                if 'queries' in sub_query_config and type(sub_query_config['queries']) is list:
+                    new_queries = new_queries + sub_query_config['queries']
+                else:
+                    print("Malformed subconfig file. Ignoring")
+            elif type(query) is dict:
+                new_queries.append(query)
+            else:
+                print("Malformed 'queries' member in config, expected either dictionaries or strings in the array. Ignoring.")
+                pass
+    config['queries'] = new_queries
+    return config
+
 if __name__ == "__main__":
     main()

src/newrelic_logging/integration.py

@@ -10,10 +10,9 @@ class DataFormat(Enum):
     LOGS = 1
     EVENTS = 2
 
-#TODO: get general api_ver that applies to all queries of one instance
 #TODO: move queries to the instance level, so we can have different queries for each instance.
 #TODO: also keep general queries that apply to all instances.
-    
+
 class Integration:
     numeric_fields_list = set()
 

src/newrelic_logging/salesforce.py

@@ -391,7 +391,7 @@ def execute_query(self, query: Query, session):
             Telemetry().log_err(f"Error while trying SOQL query: {repr(e)}")
             raise SalesforceApiException(-1, f'error when trying to run SOQL query. cause: {e}') from e
 
-    # TODO / NOTE: Is it possible that different SF orgs have overlapping IDs? If this is possible, we should use a different
+    # NOTE: Is it possible that different SF orgs have overlapping IDs? If this is possible, we should use a different
     #       database for each org, or add a prefix to keys to avoid conflicts.
 
     def download_file(self, session, url):
@@ -469,11 +469,6 @@ def is_logfile_response(self, records):
             return 'LogFile' in records[0]
         else:
             return True
-
-    # TODO: Remaining NR API limits to ensure:
-    #  - Check attribute key and value size limits (255 and 4094 bytes respectively).
-    #  - Check max number of attributes per event (255).
-    # Action: crop.
 
     def build_log_from_event(self, records, query: Query):
         logs = []