From d0611daee905715967d1ca867d596febd5117ba8 Mon Sep 17 00:00:00 2001 From: roversch Date: Mon, 9 Dec 2024 15:56:52 +0100 Subject: [PATCH] Make benchmarks great again (#45) --- Makefile | 14 +++++++----- bench/analysis/dgemm_performance.py | 27 +++++++++++------------ bench/analysis/dgemm_performance_ratio.py | 2 +- cmake/InitialCache.cmake | 6 ++--- docker/x86_64/Dockerfile | 2 +- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 76aed2b..1c779e8 100644 --- a/Makefile +++ b/Makefile @@ -5,19 +5,19 @@ BENCH_EIGEN = build/bin/bench-eigen BENCH_BLAST = build/bin/bench-blast BENCH_BLAST_OUTPUT_DIR = $(shell git rev-parse --short HEAD) BENCH_LIBXSMM = build/bin/bench-libxsmm -BENCHMARK_OPTIONS = --benchmark_repetitions=30 --benchmark_counters_tabular=true --benchmark_out_format=json --benchmark_enable_random_interleaving=true --benchmark_min_warmup_time=10 --benchmark_min_time=1000000x +BENCHMARK_OPTIONS = --benchmark_repetitions=3 --benchmark_counters_tabular=true --benchmark_out_format=json RUN_MATLAB = matlab -nodisplay -nosplash -nodesktop -r BENCH_DATA = bench_result/data BENCH_IMAGE = bench_result/image PYTHON = python3 # run-benchmark-blaze-dynamic: -# $(BENCH_TMPC) --benchmark_filter="BM_gemm_blaze_dynamic*" $(BENCHMARK_OPTIONS) \ +# $(BENCH_BLAST) --benchmark_filter="BM_gemm_blaze_dynamic*" $(BENCHMARK_OPTIONS) \ # --benchmark_out=${BENCH_DATA}/blaze-dynamic.json -${BENCH_DATA}/loop-naive.json: - $(BENCH_TMPC) --benchmark_filter="BM_gemm_loop_naive*" $(BENCHMARK_OPTIONS) \ - --benchmark_out=${BENCH_DATA}/loop-naive.json +# ${BENCH_DATA}/loop-naive.json: +# $(BENCH_BLAST) --benchmark_filter="BM_gemm_loop_naive*" $(BENCHMARK_OPTIONS) \ +# --benchmark_out=${BENCH_DATA}/loop-naive.json # # DGEMM/SGEMM bechmarks data @@ -65,18 +65,22 @@ ${BENCH_DATA}/sgemm-blaze-static.json: $(BENCH_BLAZE) ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json: $(BENCH_BLAST) $(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel" $(BENCHMARK_OPTIONS) \ --benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json + cp $@ ${BENCH_DATA} ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json: $(BENCH_BLAST) $(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_panel" $(BENCHMARK_OPTIONS) \ --benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json + cp $@ ${BENCH_DATA} ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json: $(BENCH_BLAST) $(BENCH_BLAST) --benchmark_filter="BM_gemm_static_plain" $(BENCHMARK_OPTIONS) \ --benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json + cp $@ ${BENCH_DATA} ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json: $(BENCH_BLAST) $(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_plain" $(BENCHMARK_OPTIONS) \ --benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json + cp $@ ${BENCH_DATA} ${BENCH_DATA}/sgemm-blast-static-panel.json: $(BENCH_BLAST) $(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel" $(BENCHMARK_OPTIONS) \ diff --git a/bench/analysis/dgemm_performance.py b/bench/analysis/dgemm_performance.py index e64e914..e5f159f 100644 --- a/bench/analysis/dgemm_performance.py +++ b/bench/analysis/dgemm_performance.py @@ -2,8 +2,6 @@ matplotlib.use("Agg") import matplotlib.pyplot as plt import json -import glob -import pathlib def filter_aggregate(benchmarks, name): result = [] @@ -16,30 +14,31 @@ def filter_aggregate(benchmarks, name): return result +def load_benchmark(file_name): + with open(file_name) as f: + return filter_aggregate(json.load(f)['benchmarks'], 'median') + + factor = 1e+9 # Giga style = 'o-' line_width = 1 -# prop_cycle = plt.rcParams['axes.prop_cycle'] -# c = prop_cycle.by_key()['color'] - plots = [ - # {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'}, - # {'data_file': 'dgemm-mkl.json', 'label': 'MKL'}, + {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'}, + {'data_file': 'dgemm-mkl.json', 'label': 'MKL'}, {'data_file': 'dgemm-blasfeo.json', 'label': 'BLASFEO'}, # {'data_file': 'dgemm-blasfeo-blas.json', 'label': 'BLASFEO*'}, - # {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'}, + {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'}, # {'data_file': 'dgemm-eigen-dynamic.json', 'label': 'Eigen (D)'}, - # {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'}, + {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'}, # {'data_file': 'dgemm-blaze-dynamic.json', 'label': 'Blaze (D)'}, # {'data_file': 'dgemm-blaze-static.json', 'label': 'Blaze (S)'}, + {'data_file': 'dgemm-blast-static-panel.json', 'label': 'StaPan'}, + {'data_file': 'dgemm-blast-static-plain.json', 'label': 'StaPla'}, + {'data_file': 'dgemm-blast-dynamic-panel.json', 'label': 'DynPan'}, + {'data_file': 'dgemm-blast-dynamic-plain.json', 'label': 'DynPla'}, ] -for benchmark_file, benchmark_label in [('dgemm-blast-static-panel.json', 'SP'), ('dgemm-blast-static-plain.json', 'SD'), ('dgemm-blast-dynamic-panel.json', 'DP'), ('dgemm-blast-dynamic-plain.json', 'DD')]: - files = glob.glob('./**/' + benchmark_file, recursive=True, root_dir='bench_result/data') - for file in files: - plots.append({'data_file': file, 'label': f'BLAST ({benchmark_label}) {pathlib.Path(file).parent.stem}'}) - fig = plt.figure(figsize=[10, 6]) ax = fig.subplots() diff --git a/bench/analysis/dgemm_performance_ratio.py b/bench/analysis/dgemm_performance_ratio.py index 4656952..b7b96c2 100644 --- a/bench/analysis/dgemm_performance_ratio.py +++ b/bench/analysis/dgemm_performance_ratio.py @@ -16,7 +16,7 @@ def filter_aggregate(benchmarks, name): def load_benchmark(file_name): with open(file_name) as f: - return filter_aggregate(json.load(f)['benchmarks'], 'mean') + return filter_aggregate(json.load(f)['benchmarks'], 'median') factor = 1e+9 # Giga diff --git a/cmake/InitialCache.cmake b/cmake/InitialCache.cmake index b9f2823..ebf1540 100644 --- a/cmake/InitialCache.cmake +++ b/cmake/InitialCache.cmake @@ -1,6 +1,6 @@ -set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Build type (Debug, Release, etc.)") -set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx -mavx2 -msse4 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3'" CACHE STRING "Recommended compiler flags") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build") +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type (Debug, Release, etc.)") +set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx2 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3'" CACHE STRING "Recommended compiler flags") +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build") set(BLAST_WITH_TEST ON CACHE BOOL "Build blast tests") set(BLAST_WITH_BENCHMARK ON CACHE BOOL "Build blast benchmarks") set(BLAST_WITH_BLASFEO ON CACHE BOOL "Build blasfeo C++ interface") diff --git a/docker/x86_64/Dockerfile b/docker/x86_64/Dockerfile index 6b21c22..790ee0e 100644 --- a/docker/x86_64/Dockerfile +++ b/docker/x86_64/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:kinetic +FROM ubuntu:noble WORKDIR /root RUN apt-get update RUN apt-get upgrade -y