Skip to content

Commit

Permalink
Updates bless perf flags
Browse files Browse the repository at this point in the history
  • Loading branch information
@jasonb5
jasonb5 committed Oct 30, 2023
1 parent 2f6d7c1 commit 9227c53
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 14 deletions.
16 changes: 11 additions & 5 deletions CIME/Tools/bless_test_results
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,6 @@ def create_bless_options(parser):

mutual_bless_group = bless_group.add_mutually_exclusive_group()

mutual_std_group = mutual_bless_group.add_mutually_exclusive_group()

mutual_bless_group.add_argument(
"-n", "--namelists-only", action="store_true", help="Only analyze namelists."
)
Expand All @@ -118,14 +116,22 @@ def create_bless_options(parser):
"--hist-only", action="store_true", help="Only analyze history files."
)

mutual_perf_group = mutual_bless_group.add_mutually_exclusive_group()
mutual_perf_group = bless_group.add_mutually_exclusive_group()

mutual_perf_group.add_argument(
"--bless-throughput", action="store_true", help="Bless throughput"
"--bless-tput",
action="store_true",
help="Bless throughput, use `--bless-perf` to bless throughput and memory",
)

mutual_perf_group.add_argument(
"--bless-memory", action="store_true", help="Bless memory"
"--bless-mem",
action="store_true",
help="Bless memory, use `--bless-perf` to bless throughput and memory",
)

bless_group.add_argument(
"--bless-perf", action="store_true", help="Bless both throughput and memory"
)


Expand Down
17 changes: 12 additions & 5 deletions CIME/bless_test_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,11 +211,12 @@ def bless_test_results(
new_test_root=None,
new_test_id=None,
exclude=None,
bless_throughput=False,
bless_memory=False,
bless_tput=False,
bless_mem=False,
bless_perf=False,
**_, # Capture all for extra
):
bless_all = not ((namelists_only | hist_only) or (bless_throughput | bless_memory))
bless_all = not (namelists_only | hist_only)

test_status_files = get_test_status_files(test_root, compiler, test_id=test_id)

Expand Down Expand Up @@ -308,13 +309,13 @@ def bless_test_results(
if hist_only or bless_all:
hist_bless = bless_needed

if bless_throughput:
if bless_tput or bless_perf:
tput_bless = bless_needed

if not tput_bless:
tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS

if bless_memory:
if bless_mem or bless_perf:
mem_bless = bless_needed

if not mem_bless:
Expand All @@ -328,6 +329,12 @@ def bless_test_results(
)
)
else:
logger.debug("Determined blesses for {!r}".format(test_name))
logger.debug("nl_bless = {}".format(nl_bless))
logger.debug("hist_bless = {}".format(hist_bless))
logger.debug("tput_bless = {}".format(tput_bless))
logger.debug("mem_bless = {}".format(mem_bless))

logger.info(
"###############################################################################"
)
Expand Down
55 changes: 51 additions & 4 deletions CIME/tests/test_unit_bless_test_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,48 @@ def test_bless_throughput(self, perf_compare_throughput_baseline):

assert success

@mock.patch("CIME.bless_test_results._bless_throughput")
@mock.patch("CIME.bless_test_results._bless_memory")
@mock.patch("CIME.bless_test_results.Case")
@mock.patch("CIME.bless_test_results.TestStatus")
@mock.patch("CIME.bless_test_results.get_test_status_files")
def test_bless_perf(
self,
get_test_status_files,
TestStatus,
Case,
_bless_memory,
_bless_throughput,
):
get_test_status_files.return_value = [
"/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus",
]

ts = TestStatus.return_value
ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu"
ts.get_overall_test_status.return_value = ("PASS", "RUN")
ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL", "FAIL"]

case = Case.return_value.__enter__.return_value

_bless_memory.return_value = (True, "")

_bless_throughput.return_value = (True, "")

success = bless_test_results(
"master",
"/tmp/baseline",
"/tmp/cases",
"gnu",
force=True,
bless_perf=True,
)

assert success
_bless_memory.assert_called()
_bless_throughput.assert_called()

@mock.patch("CIME.bless_test_results._bless_throughput")
@mock.patch("CIME.bless_test_results._bless_memory")
@mock.patch("CIME.bless_test_results.Case")
@mock.patch("CIME.bless_test_results.TestStatus")
Expand All @@ -419,6 +461,7 @@ def test_bless_memory_only(
TestStatus,
Case,
_bless_memory,
_bless_throughput,
):
get_test_status_files.return_value = [
"/tmp/cases/SMS.f19_g16.S.docker_gnu/TestStatus",
Expand All @@ -427,7 +470,7 @@ def test_bless_memory_only(
ts = TestStatus.return_value
ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu"
ts.get_overall_test_status.return_value = ("PASS", "RUN")
ts.get_status.side_effect = ["PASS", "FAIL"]
ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL"]

case = Case.return_value.__enter__.return_value

Expand All @@ -439,13 +482,15 @@ def test_bless_memory_only(
"/tmp/cases",
"gnu",
force=True,
bless_memory=True,
bless_mem=True,
)

assert success
_bless_memory.assert_called()
_bless_throughput.assert_not_called()

@mock.patch("CIME.bless_test_results._bless_throughput")
@mock.patch("CIME.bless_test_results._bless_memory")
@mock.patch("CIME.bless_test_results.Case")
@mock.patch("CIME.bless_test_results.TestStatus")
@mock.patch("CIME.bless_test_results.get_test_status_files")
Expand All @@ -454,6 +499,7 @@ def test_bless_throughput_only(
get_test_status_files,
TestStatus,
Case,
_bless_memory,
_bless_throughput,
):
get_test_status_files.return_value = [
Expand All @@ -463,7 +509,7 @@ def test_bless_throughput_only(
ts = TestStatus.return_value
ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu"
ts.get_overall_test_status.return_value = ("PASS", "RUN")
ts.get_status.side_effect = ["PASS", "PASS", "FAIL"]
ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL"]

case = Case.return_value.__enter__.return_value

Expand All @@ -475,10 +521,11 @@ def test_bless_throughput_only(
"/tmp/cases",
"gnu",
force=True,
bless_throughput=True,
bless_tput=True,
)

assert success
_bless_memory.assert_not_called()
_bless_throughput.assert_called()

@mock.patch("CIME.bless_test_results.bless_namelists")
Expand Down

0 comments on commit 9227c53

Please sign in to comment.