From 4228967b84016f6fd45ed6426ec8d5780f168c14 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 5 Dec 2024 08:40:19 -0700 Subject: [PATCH 1/4] Add support for "imp"-style finders in the DeferredImportCallbackFinder --- pyomo/common/dependencies.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pyomo/common/dependencies.py b/pyomo/common/dependencies.py index 08b9c6ca72c..68a6ed79d59 100644 --- a/pyomo/common/dependencies.py +++ b/pyomo/common/dependencies.py @@ -11,6 +11,7 @@ import inspect import importlib +import importlib.util import logging import sys import warnings @@ -520,13 +521,22 @@ def find_spec(self, fullname, path, target=None): spec = None # Continue looking for the finder that would have originally - # loaded the deferred import module b starting at the next + # loaded the deferred import module by starting at the next # finder in sys.meta_path (this way, we are agnostic to where # the module is coming from: file system, registry, etc.) for finder in sys.meta_path[sys.meta_path.index(self) + 1 :]: - spec = finder.find_spec(fullname, path, target) - if spec is not None: - break + if hasattr(finder, 'find_spec'): + # Support standard importlib MetaPathFinders + spec = finder.find_spec(fullname, path, target) + if spec is not None: + break + else: + # Support for imp finders/loaders (deprecated, but + # supported through Python 3.11) + loader = finder.find_module(fullname, path) + if loader is not None: + spec = importlib.util.spec_from_loader(fullname, loader) + break else: # Module not found. Returning None will proceed to the next # finder (which will eventually raise a ModuleNotFoundError) From e7d52a3cc4aeb3d61757079eda407e5563e7f139 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 5 Dec 2024 08:41:13 -0700 Subject: [PATCH 2/4] Add testing for the DeferredImportCallbackFinder --- pyomo/common/tests/test_dependencies.py | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/pyomo/common/tests/test_dependencies.py b/pyomo/common/tests/test_dependencies.py index 8c60ab6a33c..0e0f05325ce 100644 --- a/pyomo/common/tests/test_dependencies.py +++ b/pyomo/common/tests/test_dependencies.py @@ -10,6 +10,8 @@ # ___________________________________________________________________________ import inspect +import sys +from importlib.machinery import PathFinder from io import StringIO import pyomo.common.unittest as unittest @@ -24,6 +26,7 @@ UnavailableClass, _DeferredAnd, _DeferredOr, + _DeferredImportCallbackFinder, check_min_version, dill, dill_available, @@ -248,6 +251,74 @@ def _record_avail(module, avail): self.assertFalse(avail1) self.assertEqual(ans, [True, False]) + def test_callback_on_import(self): + sys.modules.pop('pyomo.common.tests.moved', None) + ans = [] + + class ImpFinder(object): + # This is an "imp" module-style finder (deprecated in Python + # 3.4 and removed in Python 3.12, but Google Collab still + # defines finders like this) + match = '' + + def find_module(self, fullname, path=None): + if fullname != self.match: + ans.append('pass') + return None + ans.append('load') + spec = PathFinder().find_spec(fullname, path) + return spec.loader + + def load_module(self, name): + pass + + def _callback(module, avail): + ans.append(len(ans)) + + attempt_import( + 'pyomo.common.tests.moved', defer_import=True, callback=_callback + ) + self.assertEqual(ans, []) + import pyomo.common.tests.moved as m + + self.assertEqual(ans, [0]) + self.assertEqual(m.Bar.data, 42) + + sys.modules.pop('pyomo.common.tests.moved', None) + del m + attempt_import( + 'pyomo.common.tests.moved', defer_import=True, callback=_callback + ) + + try: + # Test deferring to an imp-style finder that does not match + # the target module name + _finder = ImpFinder() + sys.meta_path.insert( + sys.meta_path.index(_DeferredImportCallbackFinder) + 1, _finder + ) + import pyomo.common.tests.moved as m + + self.assertEqual(ans, [0, 'pass', 2]) + self.assertEqual(m.Bar.data, 42) + + sys.modules.pop('pyomo.common.tests.moved', None) + del m + attempt_import( + 'pyomo.common.tests.moved', defer_import=True, callback=_callback + ) + + # Test deferring to an imp-style finder that DOES match the + # target module name + _finder.match = 'pyomo.common.tests.moved' + + import pyomo.common.tests.moved as m + + self.assertEqual(ans, [0, 'pass', 2, 'load', 4]) + self.assertEqual(m.Bar.data, 42) + finally: + sys.meta_path.remove(_finder) + def test_import_exceptions(self): mod, avail = attempt_import( 'pyomo.common.tests.dep_mod_except', From 565a3f1734ae42a3b5424816a67cb3862699cf52 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 5 Dec 2024 09:09:13 -0700 Subject: [PATCH 3/4] Remove interaction among tests --- pyomo/common/tests/mod.py | 17 ++++++++++++++++ pyomo/common/tests/test_dependencies.py | 26 ++++++++++++------------- 2 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 pyomo/common/tests/mod.py diff --git a/pyomo/common/tests/mod.py b/pyomo/common/tests/mod.py new file mode 100644 index 00000000000..8e34e3dea54 --- /dev/null +++ b/pyomo/common/tests/mod.py @@ -0,0 +1,17 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ +# + +# This is a simple module used as part of testing import callbacks + + +class Foo(object): + data = 42 diff --git a/pyomo/common/tests/test_dependencies.py b/pyomo/common/tests/test_dependencies.py index 0e0f05325ce..352de62cd8d 100644 --- a/pyomo/common/tests/test_dependencies.py +++ b/pyomo/common/tests/test_dependencies.py @@ -252,7 +252,7 @@ def _record_avail(module, avail): self.assertEqual(ans, [True, False]) def test_callback_on_import(self): - sys.modules.pop('pyomo.common.tests.moved', None) + sys.modules.pop('pyomo.common.tests.mod', None) ans = [] class ImpFinder(object): @@ -276,18 +276,18 @@ def _callback(module, avail): ans.append(len(ans)) attempt_import( - 'pyomo.common.tests.moved', defer_import=True, callback=_callback + 'pyomo.common.tests.mod', defer_import=True, callback=_callback ) self.assertEqual(ans, []) - import pyomo.common.tests.moved as m + import pyomo.common.tests.mod as m self.assertEqual(ans, [0]) - self.assertEqual(m.Bar.data, 42) + self.assertEqual(m.Foo.data, 42) - sys.modules.pop('pyomo.common.tests.moved', None) + sys.modules.pop('pyomo.common.tests.mod', None) del m attempt_import( - 'pyomo.common.tests.moved', defer_import=True, callback=_callback + 'pyomo.common.tests.mod', defer_import=True, callback=_callback ) try: @@ -297,25 +297,25 @@ def _callback(module, avail): sys.meta_path.insert( sys.meta_path.index(_DeferredImportCallbackFinder) + 1, _finder ) - import pyomo.common.tests.moved as m + import pyomo.common.tests.mod as m self.assertEqual(ans, [0, 'pass', 2]) - self.assertEqual(m.Bar.data, 42) + self.assertEqual(m.Foo.data, 42) - sys.modules.pop('pyomo.common.tests.moved', None) + sys.modules.pop('pyomo.common.tests.mod', None) del m attempt_import( - 'pyomo.common.tests.moved', defer_import=True, callback=_callback + 'pyomo.common.tests.mod', defer_import=True, callback=_callback ) # Test deferring to an imp-style finder that DOES match the # target module name - _finder.match = 'pyomo.common.tests.moved' + _finder.match = 'pyomo.common.tests.mod' - import pyomo.common.tests.moved as m + import pyomo.common.tests.mod as m self.assertEqual(ans, [0, 'pass', 2, 'load', 4]) - self.assertEqual(m.Bar.data, 42) + self.assertEqual(m.Foo.data, 42) finally: sys.meta_path.remove(_finder) From 2510d06b8e33ce188d4e032fb53a4ffbc8ee6db2 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 5 Dec 2024 09:13:58 -0700 Subject: [PATCH 4/4] NFC: apply black --- pyomo/common/tests/test_dependencies.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyomo/common/tests/test_dependencies.py b/pyomo/common/tests/test_dependencies.py index 352de62cd8d..fc4af1af53b 100644 --- a/pyomo/common/tests/test_dependencies.py +++ b/pyomo/common/tests/test_dependencies.py @@ -275,9 +275,7 @@ def load_module(self, name): def _callback(module, avail): ans.append(len(ans)) - attempt_import( - 'pyomo.common.tests.mod', defer_import=True, callback=_callback - ) + attempt_import('pyomo.common.tests.mod', defer_import=True, callback=_callback) self.assertEqual(ans, []) import pyomo.common.tests.mod as m @@ -286,9 +284,7 @@ def _callback(module, avail): sys.modules.pop('pyomo.common.tests.mod', None) del m - attempt_import( - 'pyomo.common.tests.mod', defer_import=True, callback=_callback - ) + attempt_import('pyomo.common.tests.mod', defer_import=True, callback=_callback) try: # Test deferring to an imp-style finder that does not match