From b5676b0068a81afbbdbb837f102f93d88d9066f7 Mon Sep 17 00:00:00 2001 From: lupin012 <58134934+lupin012@users.noreply.github.com> Date: Fri, 22 Mar 2024 16:39:34 +0100 Subject: [PATCH] perf_test: return 1 in case of ratio non 100% and in any fail reason (#118) --- perf/run_perf_tests.py | 49 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/perf/run_perf_tests.py b/perf/run_perf_tests.py index 9fbb40cc..116980de 100755 --- a/perf/run_perf_tests.py +++ b/perf/run_perf_tests.py @@ -71,7 +71,7 @@ def usage(argv): print("-c,--run-vegeta-on-core <...> taskset format for vegeta (e.g. 0-1:2-3 or 0-2:3-4) [default: " + DEFAULT_DAEMON_VEGETA_ON_CORE +"]") print("-T,--response-timeout : vegeta response timeout [default: " + DEFAULT_VEGETA_RESPONSE_TIMEOUT + "]") print("-M,--max-body-rsp : max number of bytes to read from response bodies [default: " + DEFAULT_MAX_BODY_RSP + "]") - sys.exit(-1) + sys.exit(1) def get_process(process_name: str): """ Return the running process having specified name or None if not exists """ @@ -207,7 +207,7 @@ def __parse_args(self, argv): # print help information and exit: print(err) usage(argv) - sys.exit(-1) + sys.exit(1) class PerfTest: @@ -240,7 +240,7 @@ def copy_and_extract_pattern_file(self): """ Copy the vegeta pattern file into /tmp/run_tests_xyz/ and untar the file """ if os.path.exists(self.config.vegeta_pattern_tar_file) == 0: print ("ERROR: invalid pattern file: ", self.config.vegeta_pattern_tar_file) - sys.exit(-1) + sys.exit(1) cmd = "mkdir " + RUN_TEST_DIRNAME status = os.system(cmd) cmd = "/bin/cp -f " + self.config.vegeta_pattern_tar_file + " " + VEGETA_TAR_FILE_NAME @@ -249,7 +249,7 @@ def copy_and_extract_pattern_file(self): status = os.system(cmd) if int(status) != 0: print("Vegeta pattern copy failed. Test Aborted!") - sys.exit(-1) + sys.exit(1) cmd = "cd " + RUN_TEST_DIRNAME + "; tar xvf " + VEGETA_TAR_FILE_NAME + " > /dev/null" if self.config.tracing: @@ -257,7 +257,7 @@ def copy_and_extract_pattern_file(self): status = os.system(cmd) if int(status) != 0: print("Vegeta pattern untar failed. Test Aborted!") - sys.exit(-1) + sys.exit(1) # If address is provided substitute the address and port of daemon in the vegeta file if self.config.rpc_daemon_address != "localhost": @@ -295,8 +295,8 @@ def execute(self, test_number, name, qps_value, duration): sys.stdout.flush() status = os.system(cmd) if int(status) != 0: - print("vegeta test fails: Test Aborted!") - return 0 + print("vegeta attach fails: Test Aborted!") + return 1 while 1: time.sleep(3) @@ -309,13 +309,13 @@ def execute(self, test_number, name, qps_value, duration): if pid == "" : # the server is dead; kill vegeta and returns fails os.system("kill -2 $(ps aux | grep 'vegeta' | grep -v 'grep' | grep -v 'python' | awk '{print $2}') 2> /dev/null") - return 0 + print ("test failed: server is Dead") + return 1 pid = os.popen("ps aux | grep 'vegeta report' | grep -v 'grep' | awk '{print $2}'").read() if pid == "": - # Vegeta has completed its works, generate report and return OK - self.get_result(test_number, name, qps_value, duration) - return 1 + # Vegeta has completed its works, generate report + return self.get_result(test_number, name, qps_value, duration) def execute_sequence(self, sequence, tag): """ Execute the sequence of tests """ @@ -327,13 +327,12 @@ def execute_sequence(self, sequence, tag): test_name = "[{:d}.{:2d}] " test_name_formatted = test_name.format(test_number, test_rep+1) result = self.execute(test_name_formatted, tag, qps, duration) - if result == 0: - print("Server dead test Aborted!") - return 0 + if result == 1: + return 1 time.sleep(self.config.waiting_time) test_number = test_number + 1 print("") - return 1 + return 0 def get_result(self, test_number, daemon_name, qps_value, duration): """ Processes the report file generated by vegeta and reads latency data """ @@ -348,7 +347,7 @@ def get_result(self, test_number, daemon_name, qps_value, duration): newline = file_raws[5].replace('\n', ' ') ratio = newline.split(' ')[34] if len(file_raws) > 8: - error = file_raws[8] + error = file_raws[8].rstrip() print(" [ Ratio="+ratio+", MaxLatency="+max_latency+ " Error: " + error +"]") else: error = "" @@ -357,11 +356,17 @@ def get_result(self, test_number, daemon_name, qps_value, duration): finally: file.close() + if error != "" or ratio != "100.00%": + print ("test failed: ratio is not 100.00%") + return 1 + if self.config.create_test_report: self.test_report.write_test_report(daemon_name, test_number, threads, qps_value, duration, min_latency, latency_values[7], latency_values[8], \ latency_values[9], latency_values[10], latency_values[11], max_latency, ratio, error) os.system("/bin/rm " + test_report_filename) + return 0 + class Hardware: """ Extract hardware information from the underlying platform. """ @@ -517,27 +522,25 @@ def main(argv): if config.test_mode in ("1", "3"): result = perf_test.execute_sequence(current_sequence, SILKRPC) - if result == 0: - print("Server dead test Aborted!") + if result == 1: if config.create_test_report: test_report.close() - sys.exit(-1) + return 1 if config.test_mode == "3": print("--------------------------------------------------------------------------------------------\n") if config.test_mode in ("2", "3"): result = perf_test.execute_sequence(current_sequence, RPCDAEMON) - if result == 0: - print("Server dead test Aborted!") + if result == 1: if config.create_test_report: test_report.close() - sys.exit(-1) + return 1 if config.create_test_report: test_report.close() perf_test.cleanup(0) print("Performance Test completed successfully.") - + return 0 # # module as main