Add support for env var removal.

Commands can now have env entries with null values to indicate the
corresponding environment variable should be removed. The leading `=`
syntax removes exact matches; otherwise the env variable name is
interpreted as a regex and all matching env vars are removed.

CHANGES.md

@@ -1,5 +1,13 @@
 # Release Notes
 
+## 0.7.0
+
+This release brings support for removing env vars to command definitions. Now, in addition to
+defaulting a variable with a `"NAME": "VALUE` entry in the `"env"` object and unconditionally
+writing a variable with `"=NAME": "VALUE"`, ambient environment variables can be removed by adding
+an `"env"` entry object with a `null` value. See the [packaging guide](docs/packaging.md) for more
+details.
+
 ## 0.6.0
 
 This release brings various improvements and features whose need was fleshed out by the

Cargo.toml

@@ -5,7 +5,7 @@ members = [
 
 [package]
 name = "scie-jump"
-version = "0.6.0"
+version = "0.7.0"
 description = "The self contained interpreted executable launcher."
 authors = [
     "John Sirois <[email protected]>",

docs/packaging.md

@@ -138,7 +138,15 @@ For commands, you can specify additional command line "args" to always pass to t
 environment variables to set in the ambient runtime environment via the "env" object. An environment
 variable name that begins with "=" will have the "=" stripped and will overwrite any ambient
 environment variable of the same name. Without the leading "=" the environment variable will be set
-only if not already present in the ambient runtime environment.
+only if not already present in the ambient runtime environment. If the env entry has a `null` value,
+the corresponding environment variable will be removed. For removals, names with a leading "=" will
+have the "=" stripped and the environment variable with the corresponding name will be removed. With
+no leading "=", the name is interpreted as a regular expression and all matching environment
+variable names will be removed. For example, `"BASH_.*": null` would remove all environment
+variables whose name start with `BASH_` and `"=BASH_SOURCE": null` would just remove the
+`BASH_SOURCE` environment variable. When processing env entries, removals are done first, then
+defaults are set and finally overwrites are processed. This is regardless of the order of the env
+var entries in the lift manifest JSON document.
 
 You can also supply a list of commands under "scie.lift.boot.bindings". These commands are objects
 with the same format as the "scie.lift.boot.commands" but they are not directly runnable by the end

examples/python/lift.linux-x86_64.json

@@ -8,7 +8,8 @@
           "": {
             "description": "The Pants build tool.",
             "env": {
-              "=PATH": "{cpython}/python/bin:{scie.env.PATH}"
+              "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
+              "PEX_.*": null
             },
             "exe": "{scie.bindings.venv}/venv/bin/python3.9",
             "args": [
@@ -19,7 +20,7 @@
             "description": "A Python repl with Pants (minus plugins) available for inspection.",
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
-              "PEX_TOOLS": "1"
+              "=PYTHONPATH": null
             },
             "exe": "{scie.bindings.venv}/venv/bin/python3.9"
           }
@@ -30,7 +31,8 @@
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
               "PEX_TOOLS": "1",
-              "PEX_ROOT": "{scie.bindings}/pex_root"
+              "PEX_ROOT": "{scie.bindings}/pex_root",
+              "PEX_.*": null
             },
             "exe": "{cpython}/python/bin/python3.9",
             "args": [

examples/python/lift.macos-x86_64.json

@@ -8,7 +8,8 @@
           "": {
             "description": "The Pants build tool.",
             "env": {
-              "=PATH": "{cpython}/python/bin:{scie.env.PATH}"
+              "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
+              "PEX_.*": null
             },
             "exe": "{scie.bindings.venv}/venv/bin/python3.9",
             "args": [
@@ -19,7 +20,7 @@
             "description": "A Python repl with Pants (minus plugins) available for inspection.",
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
-              "PEX_TOOLS": "1"
+              "=PYTHONPATH": null
             },
             "exe": "{scie.bindings.venv}/venv/bin/python3.9"
           }
@@ -30,7 +31,8 @@
             "env": {
               "=PATH": "{cpython}/python/bin:{scie.env.PATH}",
               "PEX_TOOLS": "1",
-              "PEX_ROOT": "{scie.bindings}/pex_root"
+              "PEX_ROOT": "{scie.bindings}/pex_root",
+              "PEX_.*": null
             },
             "exe": "{cpython}/python/bin/python3.9",
             "args": [

examples/python/test.sh

@@ -8,13 +8,29 @@ trap gc EXIT
 "${SCIE_JUMP}" "${LIFT}"
 gc "${PWD}/pants" "${PWD}/.pants.d" "${PWD}/.pids"
 
-# Observe initial install and subsequent short-circuiting of install activity.
-time RUST_LOG=trace ./pants -V
+# Observe initial install is un-perturbed by a binding command hostile setting for PEX_TOOLS (env
+# var removal happens 1st).
+time RUST_LOG=trace PEX_TOOLS=0 ./pants -V
+
+# Observe subsequent short-circuiting of install activity.
 time RUST_LOG=debug ./pants -V
 
 # Use the built-in BusyBox functionality via env var.
 SCIE_BOOT=repl ./pants -c 'from pants.util import strutil; print(strutil.__file__)'
 
+# Verify targeted environment variable removal (=PYTHONPATH is zapped by the repl command).
+SCIE_BOOT=repl PYTHONPATH=bob ./pants -c '
+import os
+assert "PYTHONPATH" not in os.environ, (
+    f"""PYTHONPATH was not scrubbed: {os.environ["PYTHONPATH"]}"""
+)
+'
+
+# Verify regex environment variable removal (PEX_.* is zapped by the default command and without
+# that, the Pants venv PEX would try to execute foo.bar:baz instead of the `-c pants` console
+# script).
+PEX_MODULE="foo.bar:baz" ./pants -V
+
 # Confirm boot bindings re-run successfully when the lift manifest changes - which allocates a new
 # boot bindings directory.
 jq '

jump/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jump"
-version = "0.6.0"
+version = "0.7.0"
 description = "The bulk of the scie-jump binary logic."
 authors = [
     "John Sirois <[email protected]>",
@@ -19,6 +19,7 @@ itertools = "0.10"
 log = { workspace = true }
 logging_timer = { workspace = true }
 memmap = "0.7"
+regex = { version = "1.7", default_features = false, features = ["std"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.10"

jump/src/comparable_regex.rs

@@ -0,0 +1,62 @@
+// Copyright 2022 Science project contributors.
+// Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+use std::cmp::Ordering;
+use std::fmt::{Display, Formatter};
+use std::hash::{Hash, Hasher};
+use std::ops::Deref;
+
+use regex::Regex;
+
+#[derive(Clone, Debug)]
+pub struct ComparableRegex(Regex);
+
+impl Deref for ComparableRegex {
+    type Target = Regex;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Display for ComparableRegex {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.0.as_str())
+    }
+}
+
+impl PartialEq<Self> for ComparableRegex {
+    fn eq(&self, other: &Self) -> bool {
+        self.0.as_str() == other.0.as_str()
+    }
+}
+
+impl Eq for ComparableRegex {}
+
+impl PartialOrd<Self> for ComparableRegex {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        self.0.as_str().partial_cmp(other.0.as_str())
+    }
+}
+
+impl Ord for ComparableRegex {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.0.as_str().cmp(other.0.as_str())
+    }
+}
+
+impl Hash for ComparableRegex {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.0.as_str().hash(state);
+    }
+}
+
+impl TryFrom<&str> for ComparableRegex {
+    type Error = String;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        Ok(Self(Regex::new(value).map_err(|e| {
+            format!("Failed to parse {value} as a regex: {e}")
+        })?))
+    }
+}

jump/src/config.rs

@@ -208,7 +208,7 @@ pub struct Cmd {
     pub args: Vec<String>,
     #[serde(default)]
     #[serde(skip_serializing_if = "BTreeMap::is_empty")]
-    pub env: BTreeMap<EnvVar, String>,
+    pub env: BTreeMap<EnvVar, Option<String>>,
     #[serde(default)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub description: Option<String>,
@@ -414,12 +414,14 @@ mod tests {
                                 env: [
                                     (
                                         EnvVar::Default("DEFAULT".to_string()),
-                                        "default".to_string()
+                                        Some("default".to_string())
                                     ),
                                     (
                                         EnvVar::Replace("REPLACE".to_string()),
-                                        "replace".to_string()
-                                    )
+                                        Some("replace".to_string())
+                                    ),
+                                    (EnvVar::Default("PEX_.*".to_string()), None,),
+                                    (EnvVar::Replace("PEX".to_string()), None,)
                                 ]
                                 .into_iter()
                                 .collect(),
@@ -465,7 +467,9 @@ mod tests {
                                     "": {
                                         "env": {
                                             "PEX_VERBOSE": "1",
-                                            "=PATH": ".:${scie.env.PATH}"
+                                            "=PATH": ".:${scie.env.PATH}",
+                                            "PEX_.*": null,
+                                            "=PEX": null
                                         },
                                         "exe":"{python}/bin/python",
                                         "args": [

jump/src/context.rs

@@ -271,9 +271,15 @@ impl<'a> Context<'a> {
         }
         let mut vars = vec![];
         for (key, value) in cmd.env.iter() {
-            let (reified_value, needs_manifest) = self.reify_string(value)?;
-            needs_lift_manifest |= needs_manifest;
-            vars.push((EnvVar::from(key), reified_value.into()));
+            let final_value = match value {
+                Some(val) => {
+                    let (reified_value, needs_manifest) = self.reify_string(val)?;
+                    needs_lift_manifest |= needs_manifest;
+                    Some(reified_value)
+                }
+                None => None,
+            };
+            vars.push(EnvVar::try_from((key, final_value))?);
         }
 
         let process = Process {

jump/src/lib.rs

@@ -9,6 +9,7 @@ extern crate structure;
 
 mod archive;
 mod atomic;
+mod comparable_regex;
 pub mod config;
 mod context;
 pub mod fingerprint;