Feat/optimize csv parsing

newrelic.ini

@@ -0,0 +1,256 @@
+# ---------------------------------------------------------------------------
+
+#
+# This file configures the New Relic Python Agent.
+#
+# The path to the configuration file should be supplied to the function
+# newrelic.agent.initialize() when the agent is being initialized.
+#
+# The configuration file follows a structure similar to what you would
+# find for Microsoft Windows INI files. For further information on the
+# configuration file format see the Python ConfigParser documentation at:
+#
+#    https://docs.python.org/library/configparser.html
+#
+# For further discussion on the behaviour of the Python agent that can
+# be configured via this configuration file see:
+#
+#    https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/
+#
+
+# ---------------------------------------------------------------------------
+
+# Here are the settings that are common to all environments.
+
+[newrelic]
+
+# You must specify the license key associated with your New
+# Relic account. This may also be set using the NEW_RELIC_LICENSE_KEY
+# environment variable. This key binds the Python Agent's data to
+# your account in the New Relic service. For more information on
+# storing and generating license keys, see
+# https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#ingest-license-key
+#license_key = [YOUR LICENSE KEY]
+
+# The application name. Set this to be the name of your
+# application as you would like it to show up in New Relic UI.
+# You may also set this using the NEW_RELIC_APP_NAME environment variable.
+# The UI will then auto-map instances of your application into a
+# entry on your home dashboard page. You can also specify multiple
+# app names to group your aggregated data. For further details,
+# please see:
+# https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/app-naming/use-multiple-names-app/
+app_name = Salesforce Eventlogfile Integration
+
+# When "true", the agent collects performance data about your
+# application and reports this data to the New Relic UI at
+# newrelic.com. This global switch is normally overridden for
+# each environment below. It may also be set using the
+# NEW_RELIC_MONITOR_MODE environment variable.
+monitor_mode = true
+
+# Sets the name of a file to log agent messages to. Whatever you
+# set this to, you must ensure that the permissions for the
+# containing directory and the file itself are correct, and
+# that the user that your web application runs as can write out
+# to the file. If not able to out a log file, it is also
+# possible to say "stderr" and output to standard error output.
+# This would normally result in output appearing in your web
+# server log. It can also be set using the NEW_RELIC_LOG
+# environment variable.
+log_file = stdout
+
+# Sets the level of detail of messages sent to the log file, if
+# a log file location has been provided. Possible values, in
+# increasing order of detail, are: "critical", "error", "warning",
+# "info" and "debug". When reporting any agent issues to New
+# Relic technical support, the most useful setting for the
+# support engineers is "debug". However, this can generate a lot
+# of information very quickly, so it is best not to keep the
+# agent at this level for longer than it takes to reproduce the
+# problem you are experiencing. This may also be set using the
+# NEW_RELIC_LOG_LEVEL environment variable.
+log_level = info
+
+# High Security Mode enforces certain security settings, and prevents
+# them from being overridden, so that no sensitive data is sent to New
+# Relic. Enabling High Security Mode means that request parameters are
+# not collected and SQL can not be sent to New Relic in its raw form.
+# To activate High Security Mode, it must be set to 'true' in this
+# local .ini configuration file AND be set to 'true' in the
+# server-side configuration in the New Relic user interface. It can
+# also be set using the NEW_RELIC_HIGH_SECURITY environment variable.
+# For details, see
+# https://docs.newrelic.com/docs/subscriptions/high-security
+high_security = false
+
+# The Python Agent will attempt to connect directly to the New
+# Relic service. If there is an intermediate firewall between
+# your host and the New Relic service that requires you to use a
+# HTTP proxy, then you should set both the "proxy_host" and
+# "proxy_port" settings to the required values for the HTTP
+# proxy. The "proxy_user" and "proxy_pass" settings should
+# additionally be set if proxy authentication is implemented by
+# the HTTP proxy. The "proxy_scheme" setting dictates what
+# protocol scheme is used in talking to the HTTP proxy. This
+# would normally always be set as "http" which will result in the
+# agent then using a SSL tunnel through the HTTP proxy for end to
+# end encryption.
+# See https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy
+# for information on proxy configuration via environment variables.
+# proxy_scheme = http
+# proxy_host = hostname
+# proxy_port = 8080
+# proxy_user =
+# proxy_pass =
+
+# Capturing request parameters is off by default. To enable the
+# capturing of request parameters, first ensure that the setting
+# "attributes.enabled" is set to "true" (the default value), and
+# then add "request.parameters.*" to the "attributes.include"
+# setting. For details about attributes configuration, please
+# consult the documentation.
+# attributes.include = request.parameters.*
+
+# The transaction tracer captures deep information about slow
+# transactions and sends this to the UI on a periodic basis. The
+# transaction tracer is enabled by default. Set this to "false"
+# to turn it off.
+transaction_tracer.enabled = true
+
+# Threshold in seconds for when to collect a transaction trace.
+# When the response time of a controller action exceeds this
+# threshold, a transaction trace will be recorded and sent to
+# the UI. Valid values are any positive float value, or (default)
+# "apdex_f", which will use the threshold for a dissatisfying
+# Apdex controller action - four times the Apdex T value.
+transaction_tracer.transaction_threshold = apdex_f
+
+# When the transaction tracer is on, SQL statements can
+# optionally be recorded. The recorder has three modes, "off"
+# which sends no SQL, "raw" which sends the SQL statement in its
+# original form, and "obfuscated", which strips out numeric and
+# string literals.
+transaction_tracer.record_sql = obfuscated
+
+# Threshold in seconds for when to collect stack trace for a SQL
+# call. In other words, when SQL statements exceed this
+# threshold, then capture and send to the UI the current stack
+# trace. This is helpful for pinpointing where long SQL calls
+# originate from in an application.
+transaction_tracer.stack_trace_threshold = 0.5
+
+# Determines whether the agent will capture query plans for slow
+# SQL queries. Only supported in MySQL and PostgreSQL. Set this
+# to "false" to turn it off.
+transaction_tracer.explain_enabled = true
+
+# Threshold for query execution time below which query plans
+# will not not be captured. Relevant only when "explain_enabled"
+# is true.
+transaction_tracer.explain_threshold = 0.5
+
+# Space separated list of function or method names in form
+# 'module:function' or 'module:class.function' for which
+# additional function timing instrumentation will be added.
+transaction_tracer.function_trace =
+
+# The error collector captures information about uncaught
+# exceptions or logged exceptions and sends them to UI for
+# viewing. The error collector is enabled by default. Set this
+# to "false" to turn it off. For more details on errors, see
+# https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/agent-data/manage-errors-apm-collect-ignore-or-mark-expected/
+error_collector.enabled = true
+
+# To stop specific errors from reporting to the UI, set this to
+# a space separated list of the Python exception type names to
+# ignore. The exception name should be of the form 'module:class'.
+error_collector.ignore_classes =
+
+# Expected errors are reported to the UI but will not affect the
+# Apdex or error rate. To mark specific errors as expected, set this
+# to a space separated list of the Python exception type names to
+# expected. The exception name should be of the form 'module:class'.
+error_collector.expected_classes =
+
+# Browser monitoring is the Real User Monitoring feature of the UI.
+# For those Python web frameworks that are supported, this
+# setting enables the auto-insertion of the browser monitoring
+# JavaScript fragments.
+browser_monitoring.auto_instrument = true
+
+# A thread profiling session can be scheduled via the UI when
+# this option is enabled. The thread profiler will periodically
+# capture a snapshot of the call stack for each active thread in
+# the application to construct a statistically representative
+# call tree. For more details on the thread profiler tool, see
+# https://docs.newrelic.com/docs/apm/apm-ui-pages/events/thread-profiler-tool/
+thread_profiler.enabled = true
+
+# Your application deployments can be recorded through the
+# New Relic REST API. To use this feature provide your API key
+# below then use the `newrelic-admin record-deploy` command.
+# This can also be set using the NEW_RELIC_API_KEY
+# environment variable.
+# api_key =
+
+# Distributed tracing lets you see the path that a request takes
+# through your distributed system. For more information, please
+# consult our distributed tracing planning guide.
+# https://docs.newrelic.com/docs/transition-guide-distributed-tracing
+distributed_tracing.enabled = true
+
+# This setting enables log decoration, the forwarding of log events,
+# and the collection of logging metrics if these sub-feature
+# configurations are also enabled. If this setting is false, no
+# logging instrumentation features are enabled. This can also be
+# set using the NEW_RELIC_APPLICATION_LOGGING_ENABLED environment
+# variable.
+# application_logging.enabled = true
+
+# If true, the agent captures log records emitted by your application
+# and forwards them to New Relic. `application_logging.enabled` must
+# also be true for this setting to take effect. You can also set
+# this using the NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED
+# environment variable.
+# application_logging.forwarding.enabled = true
+
+# If true, the agent decorates logs with metadata to link to entities,
+# hosts, traces, and spans. `application_logging.enabled` must also
+# be true for this setting to take effect. This can also be set
+# using the NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED
+# environment variable.
+# application_logging.local_decorating.enabled = true
+
+# If true, the agent captures metrics related to the log lines
+# being sent up by your application. This can also be set
+# using the NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED
+# environment variable.
+# application_logging.metrics.enabled = true
+
+startup_timeout = 10.0
+
+# ---------------------------------------------------------------------------
+
+#
+# The application environments. These are specific settings which
+# override the common environment settings. The settings related to a
+# specific environment will be used when the environment argument to the
+# newrelic.agent.initialize() function has been defined to be either
+# "development", "test", "staging" or "production".
+#
+
+[newrelic:development]
+monitor_mode = false
+
+[newrelic:test]
+monitor_mode = false
+
+[newrelic:staging]
+app_name = Python Application (Staging)
+monitor_mode = true
+
+[newrelic:production]
+monitor_mode = true
+
+# ---------------------------------------------------------------------------

requirements.txt

@@ -14,4 +14,5 @@ cryptography==41.0.6
 pip==23.3
 wheel==0.38.1
 setuptools==65.5.1
-future~=0.18.2
+future~=0.18.2
+newrelic==9.7.0

src/__main__.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+import newrelic.agent
+newrelic.agent.initialize('./newrelic.ini')
+
 import optparse
 import os
 import sys
@@ -10,7 +13,15 @@
 from apscheduler.schedulers.background import BlockingScheduler
 from pytz import utc
 from yaml import Loader, load
+from newrelic_logging.api import ApiFactory
+from newrelic_logging.auth import AuthenticatorFactory
+from newrelic_logging.cache import CacheFactory, BackendFactory
 from newrelic_logging.config import Config, getenv
+from newrelic_logging.newrelic import NewRelicFactory
+from newrelic_logging.pipeline import PipelineFactory
+from newrelic_logging.query import QueryFactory
+from newrelic_logging.salesforce import SalesForceFactory
+
 from newrelic_logging.integration import Integration
 from newrelic_logging.telemetry import print_info, print_warn
 
@@ -124,8 +135,16 @@ def run_once(
     event_type_fields_mapping: dict,
     numeric_fields_list: set
 ):
+
     Integration(
         config,
+        AuthenticatorFactory(),
+        CacheFactory(BackendFactory()),
+        PipelineFactory(),
+        SalesForceFactory(),
+        ApiFactory(),
+        QueryFactory(),
+        NewRelicFactory(),
         event_type_fields_mapping,
         numeric_fields_list,
         config.get_int(CRON_INTERVAL_MINUTES, 60),
@@ -154,6 +173,12 @@ def run_as_service(
     scheduler.add_job(
         Integration(
             config,
+            AuthenticatorFactory(),
+            CacheFactory(BackendFactory()),
+            PipelineFactory(),
+            SalesForceFactory(),
+            QueryFactory(),
+            NewRelicFactory(),
             event_type_fields_mapping,
             numeric_fields_list,
             0
@@ -179,7 +204,7 @@ def run(
 
     run_as_service(config, event_type_fields_mapping, numeric_fields_list)
 
-
+@newrelic.agent.background_task()
 def main():
     print_info(f'Integration start. Using program arguments {sys.argv[1:]}')
 

src/newrelic_logging/__init__.py

@@ -1,6 +1,38 @@
+from enum import Enum
+
+
 # Integration definitions
 
 VERSION         = "1.0.0"
 NAME            = "salesforce-eventlogfile"
 PROVIDER        = "newrelic-labs"
 COLLECTOR_NAME  = "newrelic-logs-salesforce-eventlogfile"
+
+
+class DataFormat(Enum):
+    LOGS = 1
+    EVENTS = 2
+
+
+class ConfigException(Exception):
+    def __init__(self, prop_name: str = None, *args: object):
+        self.prop_name = prop_name
+        super().__init__(*args)
+
+
+class LoginException(Exception):
+    pass
+
+
+class SalesforceApiException(Exception):
+    def __init__(self, err_code: int = 0, *args: object):
+        self.err_code = err_code
+        super().__init__(*args)
+
+
+class CacheException(Exception):
+    pass
+
+
+class NewRelicApiException(Exception):
+    pass