Fix local scoped package reporting (#837)

* Include isort stdlibs for determining stdlib modules

* Use isort & sys to eliminate std & builtin modules

Previously, the logic would fail to identify third party modules installed within the
local user socpe. This fixes that issue by skipping builtin and stdlib modules by name,
instead of attempting to identify third party modules based on file paths.

* Handle importlib_metadata.version being a callable

* Add isort into third party notices

* [Mega-Linter] Apply linters fixes

* Remove Python 2.7 and pypy2 testing (#835)

* Change setup-python to @v2 for py2.7

* Remove py27 and pypy testing

* Fix syntax errors

* Fix comma related syntax errors

* Fix more issues in tox

* Remove gearman test

* Containerized CI Pipeline (#836)

* Revert "Remove Python 2.7 and pypy2 testing (#835)"

This reverts commit abb6405.

* Containerize CI process

* Publish new docker container for CI images

* Rename github actions job

* Copyright tag scripts

* Drop debug line

* Swap to new CI image

* Move pip install to just main python

* Remove libcurl special case from tox

* Install special case packages into main image

* Remove unused packages

* Remove all other triggers besides manual

* Add make run command

* Cleanup small bugs

* Fix CI Image Tagging (#838)

* Correct templated CI image name

* Pin pypy2.7 in image

* Fix up scripting

* Temporarily Restore Old CI Pipeline (#841)

* Restore old pipelines

* Remove python 2 from setup-python

* Rework CI Pipeline (#839)

Change pypy to pypy27 in tox.

Fix checkout logic

Pin tox requires

* Fix Tests on New CI (#843)

* Remove non-root user

* Test new CI image

* Change pypy to pypy27 in tox.

* Fix checkout logic

* Fetch git tags properly

* Pin tox requires

* Adjust default db settings for github actions

* Rename elasticsearch services

* Reset to new pipelines

* [Mega-Linter] Apply linters fixes

* Fix timezone

* Fix docker networking

* Pin dev image to new sha

* Standardize gearman DB settings

* Fix elasticsearch settings bug

* Fix gearman bug

* Add missing odbc headers

* Add more debug messages

* Swap out dev ci image

* Fix required virtualenv version

* Swap out dev ci image

* Swap out dev ci image

* Remove aioredis v1 for EOL

* Add coverage paths for docker container

* Unpin ci container

---------

Co-authored-by: TimPansino <TimPansino@users.noreply.github.com>

* Trigger tests

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: hmstepanek <hmstepanek@users.noreply.github.com>
Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com>
Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com>
Co-authored-by: TimPansino <TimPansino@users.noreply.github.com>
Co-authored-by: Uma Annamalai <uannamalai@newrelic.com>

MANIFEST.in

@@ -8,3 +8,4 @@ include newrelic/common/cacert.pem
 include newrelic/packages/wrapt/LICENSE
 include newrelic/packages/wrapt/README
 include newrelic/packages/urllib3/LICENSE.txt
+include newrelic/packages/isort/LICENSE

THIRD_PARTY_NOTICES.md

@@ -14,7 +14,16 @@ Copyright (c) Django Software Foundation and individual contributors.
 
 Distributed under the following license(s):
 
-  * [The BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause)
+* [The BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause)
+
+
+## [isort](https://pypi.org/project/isort)
+
+Copyright (c) 2013 Timothy Edmund Crosley
+
+Distributed under the following license(s):
+
+* [The MIT License](http://opensource.org/licenses/MIT)
 
 
 ## [six](https://pypi.org/project/six)
@@ -23,7 +32,7 @@ Copyright (c) 2010-2013 Benjamin Peterson
 
 Distributed under the following license(s):
 
-  * [The MIT License](http://opensource.org/licenses/MIT)
+* [The MIT License](http://opensource.org/licenses/MIT)
 
 
 ## [time.monotonic](newrelic/common/_monotonic.c)
@@ -32,7 +41,7 @@ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
 
 Distributed under the following license(s):
 
-  * [Python Software Foundation](https://docs.python.org/3/license.html)
+* [Python Software Foundation](https://docs.python.org/3/license.html)
 
 
 ## [urllib3](https://pypi.org/project/urllib3)
@@ -41,7 +50,7 @@ Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 
 Distributed under the following license(s):
 
-  * [The MIT License](http://opensource.org/licenses/MIT)
+* [The MIT License](http://opensource.org/licenses/MIT)
 
 
 ## [wrapt](https://pypi.org/project/wrapt)
@@ -51,5 +60,5 @@ All rights reserved.
 
 Distributed under the following license(s):
 
-  * [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause)
+* [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause)
 

newrelic/common/package_version_utils.py

@@ -73,6 +73,10 @@ def _get_package_version(name):
     for attr in VERSION_ATTRS:
         try:
             version = getattr(module, attr, None)
+            # In certain cases like importlib_metadata.version, version is a callable
+            # function.
+            if callable(version):
+                continue
             # Cast any version specified as a list into a tuple.
             version = tuple(version) if isinstance(version, list) else version
             if version not in NULL_VERSIONS:
@@ -95,4 +99,4 @@ def _get_package_version(name):
             if version not in NULL_VERSIONS:
                 return version
         except Exception:
-            pass
+            pass

newrelic/core/environment.py

@@ -17,10 +17,10 @@
 
 """
 
+import logging
 import os
 import platform
 import sys
-import sysconfig
 
 import newrelic
 from newrelic.common.package_version_utils import get_package_version
@@ -29,12 +29,15 @@
     physical_processor_count,
     total_physical_memory,
 )
+from newrelic.packages.isort import stdlibs as isort_stdlibs
 
 try:
     import newrelic.core._thread_utilization
 except ImportError:
     pass
 
+_logger = logging.getLogger(__name__)
+
 
 def environment_settings():
     """Returns an array of arrays of environment settings"""
@@ -195,8 +198,7 @@ def environment_settings():
     env.extend(dispatcher)
 
     # Module information.
-    purelib = sysconfig.get_path("purelib")
-    platlib = sysconfig.get_path("platlib")
+    stdlib_builtin_module_names = _get_stdlib_builtin_module_names()
 
     plugins = []
 
@@ -208,29 +210,50 @@ def environment_settings():
     # list
     for name, module in sys.modules.copy().items():
         # Exclude lib.sub_paths as independent modules except for newrelic.hooks.
-        if "." in name and not name.startswith("newrelic.hooks."):
+        nr_hook = name.startswith("newrelic.hooks.")
+        if "." in name and not nr_hook or name.startswith("_"):
             continue
+
         # If the module isn't actually loaded (such as failed relative imports
         # in Python 2.7), the module will be None and should not be reported.
         if not module:
             continue
+
         # Exclude standard library/built-in modules.
-        # Third-party modules can be installed in either purelib or platlib directories.
-        # See https://docs.python.org/3/library/sysconfig.html#installation-paths.
-        if (
-            not hasattr(module, "__file__")
-            or not module.__file__
-            or not module.__file__.startswith(purelib)
-            or not module.__file__.startswith(platlib)
-        ):
+        if name in stdlib_builtin_module_names:
             continue
 
         try:
             version = get_package_version(name)
-            plugins.append("%s (%s)" % (name, version))
         except Exception:
-            plugins.append(name)
+            version = None
+
+        # If it has no version it's likely not a real package so don't report it unless
+        # it's a new relic hook.
+        if version or nr_hook:
+            plugins.append("%s (%s)" % (name, version))
 
     env.append(("Plugin List", plugins))
 
     return env
+
+
+def _get_stdlib_builtin_module_names():
+    builtins = set(sys.builtin_module_names)
+    # Since sys.stdlib_module_names is not available in versions of python below 3.10,
+    # use isort's hardcoded stdlibs instead.
+    python_version = sys.version_info[0:2]
+    if python_version < (3,):
+        stdlibs = isort_stdlibs.py27.stdlib
+    elif (3, 7) <= python_version < (3, 8):
+        stdlibs = isort_stdlibs.py37.stdlib
+    elif python_version < (3, 9):
+        stdlibs = isort_stdlibs.py38.stdlib
+    elif python_version < (3, 10):
+        stdlibs = isort_stdlibs.py39.stdlib
+    elif python_version >= (3, 10):
+        stdlibs = sys.stdlib_module_names
+    else:
+        _logger.warn("Unsupported Python version. Unable to determine stdlibs.")
+        return builtins
+    return builtins | stdlibs

newrelic/packages/isort/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Timothy Edmund Crosley
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

newrelic/packages/isort/stdlibs/__init__.py

@@ -0,0 +1,2 @@
+from . import all as _all
+from . import py2, py3, py27, py36, py37, py38, py39, py310, py311

newrelic/packages/isort/stdlibs/all.py

@@ -0,0 +1,3 @@
+from . import py2, py3
+
+stdlib = py2.stdlib | py3.stdlib

newrelic/packages/isort/stdlibs/py2.py

@@ -0,0 +1,3 @@
+from . import py27
+
+stdlib = py27.stdlib