Merge pull request #4542 from jedwards4b/reformat_config_machines

Reformat config machines

CIME/XML/files.py

@@ -136,7 +136,9 @@ def set_value(self, vid, value, subgroup=None, ignore_type=False):
 
     def get_schema(self, nodename, attributes=None):
         node = self.get_optional_child("entry", {"id": nodename})
+
         schemanode = self.get_optional_child("schema", root=node, attributes=attributes)
+
         if schemanode is not None:
             logger.debug("Found schema for {}".format(nodename))
             return self.get_resolved_value(self.text(schemanode))

CIME/XML/generic_xml.py

@@ -8,7 +8,7 @@
 import xml.etree.ElementTree as ET
 
 # pylint: disable=import-error
-from distutils.spawn import find_executable
+from shutil import which
 import getpass
 from copy import deepcopy
 from collections import namedtuple
@@ -105,7 +105,8 @@ def __init__(
 
     def read(self, infile, schema=None):
         """
-        Read and parse an xml file into the object
+        Read and parse an xml file into the object.  The schema variable can either be a path to an xsd schema file or
+        a dictionary of paths to files by version.
         """
         cached_read = False
         if not self.DISABLE_CACHING and infile in self._FILEMAP:
@@ -126,8 +127,10 @@ def read(self, infile, schema=None):
             logger.debug("read: {}".format(infile))
             with open(infile, "r", encoding="utf-8") as fd:
                 self.read_fd(fd)
-
-            if schema is not None and self.get_version() > 1.0:
+            version = str(self.get_version())
+            if type(schema) is dict:
+                self.validate_xml_file(infile, schema[version])
+            elif schema is not None and self.get_version() > 1.0:
                 self.validate_xml_file(infile, schema)
 
             logger.debug("File version is {}".format(str(self.get_version())))
@@ -472,7 +475,7 @@ def write(self, outfile=None, force_write=False):
         xmlstr = self.get_raw_record()
 
         # xmllint provides a better format option for the output file
-        xmllint = find_executable("xmllint")
+        xmllint = which("xmllint")
 
         if xmllint:
             if isinstance(outfile, str):
@@ -688,9 +691,14 @@ def validate_xml_file(self, filename, schema):
         """
         validate an XML file against a provided schema file using pylint
         """
-        expect(os.path.isfile(filename), "xml file not found {}".format(filename))
-        expect(os.path.isfile(schema), "schema file not found {}".format(schema))
-        xmllint = find_executable("xmllint")
+        expect(
+            filename and os.path.isfile(filename),
+            "xml file not found {}".format(filename),
+        )
+        expect(
+            schema and os.path.isfile(schema), "schema file not found {}".format(schema)
+        )
+        xmllint = which("xmllint")
 
         expect(
             xmllint and os.path.isfile(xmllint),

CIME/XML/machines.py

@@ -30,6 +30,9 @@ def __init__(
         additional directory that will be searched for a config_machines.xml file; if
         found, the contents of this file will be appended to the standard
         config_machines.xml. An empty string is treated the same as None.
+
+        The schema variable can be passed as a path to an xsd schema file or a dictionary of paths
+        with version number as keys.
         """
 
         self.machine_node = None
@@ -44,15 +47,27 @@ def __init__(
             files = Files()
         if infile is None:
             infile = files.get_value("MACHINES_SPEC_FILE")
-        schema = files.get_schema("MACHINES_SPEC_FILE")
-        logger.debug("Verifying using schema {}".format(schema))
 
         self.machines_dir = os.path.dirname(infile)
         if os.path.exists(infile):
             checked_files.append(infile)
         else:
             expect(False, f"file not found {infile}")
 
+        schema = {
+            "3.0": files.get_schema(
+                "MACHINES_SPEC_FILE", attributes={"version": "3.0"}
+            ),
+            "2.0": files.get_schema(
+                "MACHINES_SPEC_FILE", attributes={"version": "2.0"}
+            ),
+        }
+        # Before v3 there was but one choice
+        if not schema["3.0"]:
+            schema = files.get_schema("MACHINES_SPEC_FILE")
+
+        logger.debug("Verifying using schema {}".format(schema))
+
         GenericXML.__init__(self, infile, schema, read_only=read_only)
 
         # Append the contents of $HOME/.cime/config_machines.xml if it exists.
@@ -91,7 +106,7 @@ def __init__(
             machine is not None,
             f"Could not initialize machine object from {', '.join(checked_files)}. This machine is not available for the target CIME_MODEL.",
         )
-        self.set_machine(machine)
+        self.set_machine(machine, schema=schema)
 
     def get_child(self, name=None, attributes=None, root=None, err_msg=None):
         if root is None:
@@ -135,10 +150,19 @@ def list_available_machines(self):
         Return a list of machines defined for a given CIME_MODEL
         """
         machines = []
-        nodes = self.get_children("machine")
-        for node in nodes:
-            mach = self.get(node, "MACH")
-            machines.append(mach)
+        if self.get_version() < 3:
+            nodes = self.get_children("machine")
+            for node in nodes:
+                mach = self.get(node, "MACH")
+                machines.append(mach)
+        else:
+            machines = [
+                os.path.basename(f.path)
+                for f in os.scandir(self.machines_dir)
+                if f.is_dir()
+            ]
+            machines.remove("cmake_macros")
+        machines.sort()
         return machines
 
     def probe_machine_name(self, warn=True):
@@ -150,6 +174,7 @@ def probe_machine_name(self, warn=True):
         names_not_found = []
 
         nametomatch = socket.getfqdn()
+
         machine = self._probe_machine_name_one_guess(nametomatch)
 
         if machine is None:
@@ -177,10 +202,15 @@ def _probe_machine_name_one_guess(self, nametomatch):
         Find a matching regular expression for nametomatch in the NODENAME_REGEX
         field in the file. First match wins. Returns None if no match is found.
         """
+        if self.get_version() < 3:
+            return self._probe_machine_name_one_guess_v2(nametomatch)
+        else:
+            return self._probe_machine_name_one_guess_v3(nametomatch)
 
-        machine = None
-        nodes = self.get_children("machine")
+    def _probe_machine_name_one_guess_v2(self, nametomatch):
 
+        nodes = self.get_children("machine")
+        machine = None
         for node in nodes:
             machtocheck = self.get(node, "MACH")
             logger.debug("machine is " + machtocheck)
@@ -222,7 +252,53 @@ def _probe_machine_name_one_guess(self, nametomatch):
 
         return machine
 
-    def set_machine(self, machine):
+    def _probe_machine_name_one_guess_v3(self, nametomatch):
+
+        node = self.get_child("NODENAME_REGEX", root=self.root)
+
+        for child in self.get_children(root=node):
+            machtocheck = self.get(child, "MACH")
+            regex_str = self.text(child)
+            logger.debug(
+                "machine is {} regex {}, nametomatch {}".format(
+                    machtocheck, regex_str, nametomatch
+                )
+            )
+
+            if regex_str is not None:
+                # an environment variable can be used
+                if regex_str.startswith("$ENV"):
+                    machine_value = self.get_resolved_value(
+                        regex_str, allow_unresolved_envvars=True
+                    )
+                    logger.debug("machine_value is {}".format(machine_value))
+                    if not machine_value.startswith("$ENV"):
+                        try:
+                            match, this_machine = machine_value.split(":")
+                        except ValueError:
+                            expect(
+                                False,
+                                "Bad formation of NODENAME_REGEX.  Expected envvar:value, found {}".format(
+                                    regex_str
+                                ),
+                            )
+                        if match == this_machine:
+                            machine = machtocheck
+                            break
+                else:
+                    regex = re.compile(regex_str)
+                    if regex.match(nametomatch):
+                        logger.debug(
+                            "Found machine: {} matches {}".format(
+                                machtocheck, nametomatch
+                            )
+                        )
+                        machine = machtocheck
+                        break
+
+        return machine
+
+    def set_machine(self, machine, schema=None):
         """
         Sets the machine block in the Machines object
 
@@ -235,15 +311,24 @@ def set_machine(self, machine):
         CIMEError: ERROR: No machine trump found
         """
         if machine == "Query":
-            self.machine = machine
-        elif self.machine != machine or self.machine_node is None:
-            self.machine_node = super(Machines, self).get_child(
-                "machine",
-                {"MACH": machine},
-                err_msg="No machine {} found".format(machine),
+            return machine
+        elif self.get_version() == 3:
+            machines_file = os.path.join(
+                self.machines_dir, machine, "config_machines.xml"
             )
-            self.machine = machine
+            if os.path.isfile(machines_file):
+                GenericXML.read(
+                    self,
+                    machines_file,
+                    schema=schema,
+                )
+        self.machine_node = super(Machines, self).get_child(
+            "machine",
+            {"MACH": machine},
+            err_msg="No machine {} found".format(machine),
+        )
 
+        self.machine = machine
         return machine
 
     # pylint: disable=arguments-differ
@@ -292,6 +377,11 @@ def get_field_from_list(self, listname, reqval=None, attributes=None):
         """
         expect(self.machine_node is not None, "Machine object has no machine defined")
         supported_values = self.get_value(listname, attributes=attributes)
+        logger.debug(
+            "supported values for {} on {} is {}".format(
+                listname, self.machine, supported_values
+            )
+        )
         # if no match with attributes, try without
         if supported_values is None:
             supported_values = self.get_value(listname, attributes=None)

CIME/data/config/cesm/config_files.xml

@@ -43,7 +43,8 @@
     <group>case_last</group>
     <file>env_case.xml</file>
     <desc>file containing machine specifications for target model primary component (for documentation only - DO NOT EDIT)</desc>
-    <schema>$CIMEROOT/CIME/data/config/xml_schemas/config_machines.xsd</schema>
+    <schema version="2.0">$CIMEROOT/CIME/data/config/xml_schemas/config_machines.xsd</schema>
+    <schema version="3.0">$CIMEROOT/CIME/data/config/xml_schemas/config_machines_version3.xsd</schema>
   </entry>
 
   <entry id="BATCH_SPEC_FILE">