Resolve incompatibility between imp and the DeferredImportCallbackFinder

pyomo/common/dependencies.py

@@ -11,6 +11,7 @@
 
 import inspect
 import importlib
+import importlib.util
 import logging
 import sys
 import warnings
@@ -520,13 +521,22 @@ def find_spec(self, fullname, path, target=None):
 
         spec = None
         # Continue looking for the finder that would have originally
-        # loaded the deferred import module b starting at the next
+        # loaded the deferred import module by starting at the next
         # finder in sys.meta_path (this way, we are agnostic to where
         # the module is coming from: file system, registry, etc.)
         for finder in sys.meta_path[sys.meta_path.index(self) + 1 :]:
-            spec = finder.find_spec(fullname, path, target)
-            if spec is not None:
-                break
+            if hasattr(finder, 'find_spec'):
+                # Support standard importlib MetaPathFinders
+                spec = finder.find_spec(fullname, path, target)
+                if spec is not None:
+                    break
+            else:
+                # Support for imp finders/loaders (deprecated, but
+                # supported through Python 3.11)
+                loader = finder.find_module(fullname, path)
+                if loader is not None:
+                    spec = importlib.util.spec_from_loader(fullname, loader)
+                    break
         else:
             # Module not found.  Returning None will proceed to the next
             # finder (which will eventually raise a ModuleNotFoundError)

pyomo/common/tests/mod.py

@@ -0,0 +1,17 @@
+#  ___________________________________________________________________________
+#
+#  Pyomo: Python Optimization Modeling Objects
+#  Copyright (c) 2008-2024
+#  National Technology and Engineering Solutions of Sandia, LLC
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
+#  rights in this software.
+#  This software is distributed under the 3-clause BSD License.
+#  ___________________________________________________________________________
+#
+
+# This is a simple module used as part of testing import callbacks
+
+
+class Foo(object):
+    data = 42

pyomo/common/tests/test_dependencies.py

@@ -10,6 +10,8 @@
 #  ___________________________________________________________________________
 
 import inspect
+import sys
+from importlib.machinery import PathFinder
 from io import StringIO
 
 import pyomo.common.unittest as unittest
@@ -24,6 +26,7 @@
     UnavailableClass,
     _DeferredAnd,
     _DeferredOr,
+    _DeferredImportCallbackFinder,
     check_min_version,
     dill,
     dill_available,
@@ -248,6 +251,70 @@ def _record_avail(module, avail):
         self.assertFalse(avail1)
         self.assertEqual(ans, [True, False])
 
+    def test_callback_on_import(self):
+        sys.modules.pop('pyomo.common.tests.mod', None)
+        ans = []
+
+        class ImpFinder(object):
+            # This is an "imp" module-style finder (deprecated in Python
+            # 3.4 and removed in Python 3.12, but Google Collab still
+            # defines finders like this)
+            match = ''
+
+            def find_module(self, fullname, path=None):
+                if fullname != self.match:
+                    ans.append('pass')
+                    return None
+                ans.append('load')
+                spec = PathFinder().find_spec(fullname, path)
+                return spec.loader
+
+            def load_module(self, name):
+                pass
+
+        def _callback(module, avail):
+            ans.append(len(ans))
+
+        attempt_import('pyomo.common.tests.mod', defer_import=True, callback=_callback)
+        self.assertEqual(ans, [])
+        import pyomo.common.tests.mod as m
+
+        self.assertEqual(ans, [0])
+        self.assertEqual(m.Foo.data, 42)
+
+        sys.modules.pop('pyomo.common.tests.mod', None)
+        del m
+        attempt_import('pyomo.common.tests.mod', defer_import=True, callback=_callback)
+
+        try:
+            # Test deferring to an imp-style finder that does not match
+            # the target module name
+            _finder = ImpFinder()
+            sys.meta_path.insert(
+                sys.meta_path.index(_DeferredImportCallbackFinder) + 1, _finder
+            )
+            import pyomo.common.tests.mod as m
+
+            self.assertEqual(ans, [0, 'pass', 2])
+            self.assertEqual(m.Foo.data, 42)
+
+            sys.modules.pop('pyomo.common.tests.mod', None)
+            del m
+            attempt_import(
+                'pyomo.common.tests.mod', defer_import=True, callback=_callback
+            )
+
+            # Test deferring to an imp-style finder that DOES match the
+            # target module name
+            _finder.match = 'pyomo.common.tests.mod'
+
+            import pyomo.common.tests.mod as m
+
+            self.assertEqual(ans, [0, 'pass', 2, 'load', 4])
+            self.assertEqual(m.Foo.data, 42)
+        finally:
+            sys.meta_path.remove(_finder)
+
     def test_import_exceptions(self):
         mod, avail = attempt_import(
             'pyomo.common.tests.dep_mod_except',