Merge pull request #4507 from ESMCI/yet_another_btr_update

Yet another btr update

Reverts to original behavior, bless_test_results with no flags will bless hist and nml
Adds performance flags --bless-tput, --bless-mem, and --bles-perf (both tput and mem)
Test suite: pytest -vvv CIME/tests/test_unit*
Test baseline:
Test namelist changes:
Test status: [bit for bit, roundoff, climate changing]

Fixes [CIME Github issue #]
User interface changes?:
Update gh-pages html (Y/N)?:

CIME/Tools/bless_test_results

@@ -116,12 +116,22 @@ def create_bless_options(parser):
         "--hist-only", action="store_true", help="Only analyze history files."
     )
 
-    mutual_bless_group.add_argument(
-        "--tput-only", action="store_true", help="Only analyze throughput."
+    mutual_perf_group = bless_group.add_mutually_exclusive_group()
+
+    mutual_perf_group.add_argument(
+        "--bless-tput",
+        action="store_true",
+        help="Bless throughput, use `--bless-perf` to bless throughput and memory",
     )
 
-    mutual_bless_group.add_argument(
-        "--mem-only", action="store_true", help="Only analyze memory."
+    mutual_perf_group.add_argument(
+        "--bless-mem",
+        action="store_true",
+        help="Bless memory, use `--bless-perf` to bless throughput and memory",
+    )
+
+    bless_group.add_argument(
+        "--bless-perf", action="store_true", help="Bless both throughput and memory"
     )
 
 

CIME/baselines/performance.py

@@ -129,6 +129,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True):
 
             write_baseline_file(baseline_file, tput)
 
+            logger.info("Updated throughput baseline to {!s}".format(tput))
+
     if memory:
         try:
             mem = perf_get_memory(case, config)
@@ -139,6 +141,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True):
 
             write_baseline_file(baseline_file, mem)
 
+            logger.info("Updated memory usage baseline to {!s}".format(mem))
+
 
 def load_coupler_customization(case):
     """

CIME/bless_test_results.py

@@ -22,7 +22,7 @@
 logger = logging.getLogger(__name__)
 
 
-def bless_throughput(
+def _bless_throughput(
     case,
     test_name,
     baseline_root,
@@ -32,6 +32,7 @@ def bless_throughput(
 ):
     success = True
     reason = None
+    below_threshold = False
 
     baseline_dir = os.path.join(
         baseline_root, baseline_name, case.get_value("CASEBASEID")
@@ -42,20 +43,10 @@ def bless_throughput(
             case, baseline_dir=baseline_dir
         )
     except FileNotFoundError as e:
-        success = False
-
         comment = f"Could not read throughput file: {e!s}"
     except Exception as e:
-        success = False
-
         comment = f"Error comparing throughput baseline: {e!s}"
 
-    # fail early
-    if not success:
-        logger.info(comment)
-
-        return success, comment
-
     if below_threshold:
         logger.info("Throughput diff appears to have been already resolved.")
     else:
@@ -74,7 +65,7 @@ def bless_throughput(
     return success, reason
 
 
-def bless_memory(
+def _bless_memory(
     case,
     test_name,
     baseline_root,
@@ -84,6 +75,7 @@ def bless_memory(
 ):
     success = True
     reason = None
+    below_threshold = False
 
     baseline_dir = os.path.join(
         baseline_root, baseline_name, case.get_value("CASEBASEID")
@@ -94,20 +86,10 @@ def bless_memory(
             case, baseline_dir=baseline_dir
         )
     except FileNotFoundError as e:
-        success = False
-
         comment = f"Could not read memory usage file: {e!s}"
     except Exception as e:
-        success = False
-
         comment = f"Error comparing memory baseline: {e!s}"
 
-    # fail early
-    if not success:
-        logger.info(comment)
-
-        return success, comment
-
     if below_threshold:
         logger.info("Memory usage diff appears to have been already resolved.")
     else:
@@ -221,8 +203,6 @@ def bless_test_results(
     test_id=None,
     namelists_only=False,
     hist_only=False,
-    tput_only=False,
-    mem_only=False,
     report_only=False,
     force=False,
     pes_file=None,
@@ -231,9 +211,12 @@ def bless_test_results(
     new_test_root=None,
     new_test_id=None,
     exclude=None,
+    bless_tput=False,
+    bless_mem=False,
+    bless_perf=False,
     **_,  # Capture all for extra
 ):
-    bless_all = not (namelists_only | hist_only | tput_only | mem_only)
+    bless_all = not (namelists_only | hist_only)
 
     test_status_files = get_test_status_files(test_root, compiler, test_id=test_id)
 
@@ -294,15 +277,15 @@ def bless_test_results(
         excluded = exclude.match(test_name) if exclude else False
 
         if (not has_no_tests and not match_test_name) or excluded:
-            logger.info("Skipping {!r}".format(test_name))
+            logger.debug("Skipping {!r}".format(test_name))
 
             continue
 
         overall_result, phase = ts.get_overall_test_status(
             ignore_namelists=True,
             ignore_memleak=True,
-            check_throughput=True,
-            check_memory=True,
+            check_throughput=False,
+            check_memory=False,
         )
 
         # See if we need to bless namelist
@@ -326,13 +309,13 @@ def bless_test_results(
             if hist_only or bless_all:
                 hist_bless = bless_needed
 
-            if tput_only or bless_all:
+            if bless_tput or bless_perf:
                 tput_bless = bless_needed
 
                 if not tput_bless:
                     tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS
 
-            if mem_only or bless_all:
+            if bless_mem or bless_perf:
                 mem_bless = bless_needed
 
                 if not mem_bless:
@@ -346,6 +329,12 @@ def bless_test_results(
                 )
             )
         else:
+            logger.debug("Determined blesses for {!r}".format(test_name))
+            logger.debug("nl_bless     = {}".format(nl_bless))
+            logger.debug("hist_bless   = {}".format(hist_bless))
+            logger.debug("tput_bless   = {}".format(tput_bless))
+            logger.debug("mem_bless    = {}".format(mem_bless))
+
             logger.info(
                 "###############################################################################"
             )
@@ -366,8 +355,9 @@ def bless_test_results(
                 if baseline_name is None:
                     baseline_name_resolved = case.get_value("BASELINE_NAME_CMP")
                     if not baseline_name_resolved:
+                        cime_root = CIME.utils.get_cime_root()
                         baseline_name_resolved = CIME.utils.get_current_branch(
-                            repo=CIME.utils.get_cime_root()
+                            repo=cime_root
                         )
                 else:
                     baseline_name_resolved = baseline_name
@@ -423,7 +413,7 @@ def bless_test_results(
                         broken_blesses.append((test_name, reason))
 
                 if tput_bless:
-                    success, reason = bless_throughput(
+                    success, reason = _bless_throughput(
                         case,
                         test_name,
                         baseline_root_resolved,
@@ -436,7 +426,7 @@ def bless_test_results(
                         broken_blesses.append((test_name, reason))
 
                 if mem_bless:
-                    success, reason = bless_memory(
+                    success, reason = _bless_memory(
                         case,
                         test_name,
                         baseline_root_resolved,