Support locking PEP-723 requirements. (#2642)

Add a `--exe` / `--script` argument to `pex3 lock {create,sync}` to support gathering requirements and interpreter constraints from PEP-723 script metadata. This complements the support added for creating PEXes using PEP-723 metadata in #2436.
pex-tool · Jan 17, 2025 · edad1b1 · edad1b1
1 parent 3dfda00
commit edad1b1
Show file tree

Hide file tree

Showing 8 changed files with 710 additions and 75 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,26 @@
 # Release Notes
 
+## 2.29.0
+
+This release brings 1st class support for newer Pip's
+`--keyring-provider` option. Previously you could only use `keyring`
+based authentication via `--use-pip-config` and either the
+`PIP_KEYRING_PROVIDER` environment variable or Pip config files.
+Although using `--keyring-provider import` is generally unusable in the
+face of Pex hermeticity strictures, `--keyring-provider subprocess` is
+viable; just ensure you have a keyring provider on the `PATH`. You can
+read more [here][Pip-KRP-subprocess].
+
+This release also brings [PEP-723][PEP-723] support to Pex locks. You
+can now pass `pex3 lock {create,sync,update} --exe <script> ...` to
+include the PEP-723 declared script requirements in the lock.
+
+* add `--keyring-provider` flag to configure keyring-based authentication (#2592)
+* Support locking PEP-723 requirements. (#2642)
+
+[Pip-KRP-subprocess]: https://pip.pypa.io/en/stable/topics/authentication/#using-keyring-as-a-command-line-application
+[PEP-723]: https://peps.python.org/pep-0723
+
 ## 2.28.1
 
 This release upgrades `science` for use in building PEX scies with

diff --git a/RELEASE.rst b/RELEASE.rst
@@ -69,12 +69,3 @@ If you're on macOS and commit signing fails, try setting ``export GPG_TTY=$(tty)
 
 Open the Release workflow run and wait for it to go green:
 https://github.com/pex-tool/pex/actions?query=workflow%3ARelease+branch%3Av2.1.29
-
-Edit the Github Release Page
-----------------------------
-
-Open the release page for edit:
-https://github.com/pex-tool/pex/releases/edit/v2.1.29
-
-1. Copy and paste the most recent CHANGES.rst section.
-2. Adapt the syntax from RestructuredText to Markdown (e.g. remove RST links ```PR #... <...>`_``).
diff --git a/pex/bin/pex.py b/pex/bin/pex.py
@@ -31,7 +31,7 @@
 from pex.enum import Enum
 from pex.fetcher import URLFetcher
 from pex.inherit_path import InheritPath
-from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints
+from pex.interpreter_constraints import InterpreterConstraints
 from pex.layout import Layout, ensure_installed
 from pex.orderedset import OrderedSet
 from pex.pep_427 import InstallableType
@@ -56,6 +56,7 @@
     PipConfiguration,
 )
 from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
+from pex.resolve.script_metadata import apply_script_metadata
 from pex.result import Error, ResultError, catch, try_
 from pex.scie import ScieConfiguration
 from pex.targets import Targets
@@ -1163,14 +1164,16 @@ def configure_requirements_and_targets(
 
     requirement_configuration = requirement_options.configure(options)
     target_config = target_options.configure(options, pip_configuration=pip_configuration)
-    script_metadata = ScriptMetadata()
+    script_metadata = None  # type: Optional[ScriptMetadata]
 
     if options.executable and options.enable_script_metadata:
-        with open(options.executable) as fp:
-            script_metadata = ScriptMetadata.parse(fp.read(), source=fp.name)
-
+        script_metadata_application = apply_script_metadata(
+            [options.executable],
+            requirement_configuration=requirement_configuration,
+            target_configuration=target_config,
+        )
+        script_metadata = script_metadata_application.scripts[0]
         if script_metadata.dependencies:
-            requirements = OrderedSet(str(req) for req in script_metadata.dependencies)
             TRACER.log(
                 "Will resolve dependencies discovered in PEP-723 script metadata from {source}"
                 "{in_addition_to}: {dependencies}".format(
@@ -1180,34 +1183,25 @@ def configure_requirements_and_targets(
                         if requirement_configuration.has_requirements
                         else ""
                     ),
-                    dependencies=" ".join(requirements),
+                    dependencies=" ".join(
+                        OrderedSet(str(req) for req in script_metadata.dependencies)
+                    ),
                 )
             )
-            if requirement_configuration.requirements:
-                requirements.update(requirement_configuration.requirements)
-            requirement_configuration = attr.evolve(
-                requirement_configuration,
-                requirements=requirements,
-            )
+        requirement_configuration = script_metadata_application.requirement_configuration
 
         if (
             script_metadata.requires_python
             and not target_config.interpreter_configuration.interpreter_constraints
         ):
-            interpreter_constraint = InterpreterConstraint(script_metadata.requires_python)
             TRACER.log(
                 "Will target interpreters matching requires-python discovered in PEP-723 script "
                 "metadata from {source}: {interpreter_constraint}".format(
-                    source=script_metadata.source, interpreter_constraint=interpreter_constraint
+                    source=options.executable,
+                    interpreter_constraint=script_metadata.requires_python,
                 )
             )
-            target_config = attr.evolve(
-                target_config,
-                interpreter_configuration=attr.evolve(
-                    target_config.interpreter_configuration,
-                    interpreter_constraints=InterpreterConstraints((interpreter_constraint,)),
-                ),
-            )
+        target_config = script_metadata_application.target_configuration
 
     try:
         targets = target_config.resolve_targets()
@@ -1216,7 +1210,7 @@ def configure_requirements_and_targets(
     except target_configuration.InterpreterConstraintsNotSatisfied as e:
         return Error(str(e), exit_code=CANNOT_SETUP_INTERPRETER)
 
-    if script_metadata.requires_python:
+    if script_metadata and script_metadata.requires_python:
         incompatible_targets = []  # type: List[str]
         for target in targets.unique_targets():
             if not target.requires_python_applies(

diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py
@@ -63,6 +63,7 @@
 from pex.resolve.resolved_requirement import Fingerprint, Pin
 from pex.resolve.resolver_configuration import LockRepositoryConfiguration, PipConfiguration
 from pex.resolve.resolver_options import parse_lockfile
+from pex.resolve.script_metadata import ScriptMetadataApplication, apply_script_metadata
 from pex.resolve.target_configuration import InterpreterConstraintsNotSatisfied, TargetConfiguration
 from pex.result import Error, Ok, Result, try_
 from pex.sorted_tuple import SortedTuple
@@ -397,6 +398,54 @@ def _no_updates(self):
         )
 
 
+@attr.s(frozen=True)
+class LockingConfiguration(object):
+    requirement_configuration = attr.ib()  # type: RequirementConfiguration
+    target_configuration = attr.ib()  # type: TargetConfiguration
+    lock_configuration = attr.ib()  # type: LockConfiguration
+    script_metadata_application = attr.ib(default=None)  # type: Optional[ScriptMetadataApplication]
+
+    def check_scripts(self, targets):
+        # type: (Targets) -> Optional[Error]
+
+        if self.script_metadata_application is None:
+            return None
+
+        errors = []  # type: List[str]
+        for target in targets.unique_targets():
+            scripts = self.script_metadata_application.target_does_not_apply(target)
+            if scripts:
+                errors.append(
+                    "{target} is not compatible with {count} {scripts}:\n"
+                    "{script_incompatibilities}".format(
+                        target=target.render_description(),
+                        count=len(scripts),
+                        scripts=pluralize(scripts, "script"),
+                        script_incompatibilities="\n".join(
+                            "   + {source} requires Python '{requires_python}'".format(
+                                source=script.source,
+                                requires_python=script.requires_python,
+                            )
+                            for script in scripts
+                        ),
+                    )
+                )
+        if errors:
+            return Error(
+                "PEP-723 scripts were specified that are incompatible with {count} lock "
+                "{targets}:\n{errors}".format(
+                    count=len(errors),
+                    targets=pluralize(errors, "target"),
+                    errors="\n".join(
+                        "{index}. {error}".format(index=index, error=error)
+                        for index, error in enumerate(errors, start=1)
+                    ),
+                )
+            )
+
+        return None
+
+
 class Lock(OutputMixin, JsonMixin, BuildTimeCommand):
     """Operate on PEX lock files."""
 
@@ -427,6 +476,17 @@ def _add_resolve_options(cls, parser):
             title="Requirement options",
             description="Indicate which distributions should be resolved",
         )
+        options_group.add_argument(
+            "--exe",
+            "--script",
+            dest="scripts",
+            default=[],
+            action="append",
+            help=(
+                "Specify scripts with PEP-723 metadata to gather requirements and interpreter "
+                "constraints from as lock inputs."
+            ),
+        )
         requirement_options.register(options_group)
         project.register_options(
             options_group,
@@ -867,13 +927,13 @@ def _resolve_targets(
             complete_platforms=target_config.complete_platforms,
         )
 
-    def _gather_requirements(
+    def _merge_project_requirements(
         self,
+        requirement_configuration,  # type: RequirementConfiguration
         pip_configuration,  # type: PipConfiguration
         targets,  # type: Targets
     ):
         # type: (...) -> RequirementConfiguration
-        requirement_configuration = requirement_options.configure(self.options)
         group_requirements = project.get_group_requirements(self.options)
         projects = project.get_projects(self.options)
         if not projects and not group_requirements:
@@ -898,15 +958,19 @@ def _gather_requirements(
                 )
         return attr.evolve(requirement_configuration, requirements=requirements)
 
-    def _create(self):
-        # type: () -> Result
-
-        pip_configuration = resolver_options.create_pip_configuration(
-            self.options, use_system_time=False
-        )
+    def _locking_configuration(self, pip_configuration):
+        # type: (PipConfiguration) -> Union[LockingConfiguration, Error]
+        requirement_configuration = requirement_options.configure(self.options)
         target_configuration = target_options.configure(
             self.options, pip_configuration=pip_configuration
         )
+        script_metadata_application = None  # type: Optional[ScriptMetadataApplication]
+        if self.options.scripts:
+            script_metadata_application = apply_script_metadata(
+                self.options.scripts, requirement_configuration, target_configuration
+            )
+            requirement_configuration = script_metadata_application.requirement_configuration
+            target_configuration = script_metadata_application.target_configuration
         if self.options.style == LockStyle.UNIVERSAL:
             lock_configuration = LockConfiguration(
                 style=LockStyle.UNIVERSAL,
@@ -928,27 +992,43 @@ def _create(self):
                 style=self.options.style,
                 elide_unused_requires_dist=self.options.elide_unused_requires_dist,
             )
+        return LockingConfiguration(
+            requirement_configuration,
+            target_configuration,
+            lock_configuration,
+            script_metadata_application=script_metadata_application,
+        )
+
+    def _create(self):
+        # type: () -> Result
 
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
+        locking_configuration = try_(self._locking_configuration(pip_configuration))
         targets = try_(
             self._resolve_targets(
                 action="creating",
                 style=self.options.style,
-                target_configuration=target_configuration,
+                target_configuration=locking_configuration.target_configuration,
             )
         )
+        try_(locking_configuration.check_scripts(targets))
         pip_configuration = try_(
             finalize_resolve_config(
                 resolver_configuration=pip_configuration,
                 targets=targets,
                 context="lock creation",
             )
         )
-        requirement_configuration = self._gather_requirements(pip_configuration, targets)
+        requirement_configuration = self._merge_project_requirements(
+            locking_configuration.requirement_configuration, pip_configuration, targets
+        )
         dependency_config = dependency_configuration.configure(self.options)
         self._dump_lockfile(
             try_(
                 create(
-                    lock_configuration=lock_configuration,
+                    lock_configuration=locking_configuration.lock_configuration,
                     requirement_configuration=requirement_configuration,
                     targets=targets,
                     pip_configuration=pip_configuration,
@@ -1571,35 +1651,11 @@ def _sync(self):
         )
         production_assert(isinstance(resolver_configuration, LockRepositoryConfiguration))
         pip_configuration = resolver_configuration.pip_configuration
+        locking_configuration = try_(self._locking_configuration(pip_configuration))
         dependency_config = dependency_configuration.configure(self.options)
-
-        target_configuration = target_options.configure(
-            self.options, pip_configuration=pip_configuration
-        )
-        if self.options.style == LockStyle.UNIVERSAL:
-            lock_configuration = LockConfiguration(
-                style=LockStyle.UNIVERSAL,
-                requires_python=tuple(
-                    str(interpreter_constraint.requires_python)
-                    for interpreter_constraint in target_configuration.interpreter_constraints
-                ),
-                target_systems=tuple(self.options.target_systems),
-                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
-            )
-        elif self.options.target_systems:
-            return Error(
-                "The --target-system option only applies to --style {universal} locks.".format(
-                    universal=LockStyle.UNIVERSAL.value
-                )
-            )
-        else:
-            lock_configuration = LockConfiguration(
-                style=self.options.style,
-                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
-            )
-
         lock_file_path = self.options.lock
         if os.path.exists(lock_file_path):
+            lock_configuration = locking_configuration.lock_configuration
             build_configuration = pip_configuration.build_configuration
             original_lock_file = try_(parse_lockfile(self.options, lock_file_path=lock_file_path))
             lock_file = attr.evolve(
@@ -1629,8 +1685,18 @@ def _sync(self):
                     dependency_config=dependency_config,
                 )
             )
-            requirement_configuration = self._gather_requirements(
-                pip_configuration, lock_update_request.targets
+            try_(locking_configuration.check_scripts(lock_update_request.targets))
+            pip_configuration = try_(
+                finalize_resolve_config(
+                    resolver_configuration=pip_configuration,
+                    targets=lock_update_request.targets,
+                    context="lock syncing",
+                )
+            )
+            requirement_configuration = self._merge_project_requirements(
+                locking_configuration.requirement_configuration,
+                pip_configuration,
+                lock_update_request.targets,
             )
             lock_update = lock_update_request.sync(
                 requirement_configuration=requirement_configuration,
@@ -1644,13 +1710,23 @@ def _sync(self):
                 self._resolve_targets(
                     action="creating",
                     style=self.options.style,
-                    target_configuration=target_configuration,
+                    target_configuration=locking_configuration.target_configuration,
+                )
+            )
+            try_(locking_configuration.check_scripts(targets))
+            pip_configuration = try_(
+                finalize_resolve_config(
+                    resolver_configuration=pip_configuration,
+                    targets=targets,
+                    context="lock creation",
                 )
             )
-            requirement_configuration = self._gather_requirements(pip_configuration, targets)
+            requirement_configuration = self._merge_project_requirements(
+                locking_configuration.requirement_configuration, pip_configuration, targets
+            )
             lockfile = try_(
                 create(
-                    lock_configuration=lock_configuration,
+                    lock_configuration=locking_configuration.lock_configuration,
                     requirement_configuration=requirement_configuration,
                     targets=targets,
                     pip_configuration=pip_configuration,