Make benchmarks great again (#45)

mkatliar · Dec 9, 2024 · d0611da · d0611da
1 parent 5886fd3
commit d0611da
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 24 deletions.
diff --git a/Makefile b/Makefile
@@ -5,19 +5,19 @@ BENCH_EIGEN = build/bin/bench-eigen
 BENCH_BLAST = build/bin/bench-blast
 BENCH_BLAST_OUTPUT_DIR = $(shell git rev-parse --short HEAD)
 BENCH_LIBXSMM = build/bin/bench-libxsmm
-BENCHMARK_OPTIONS = --benchmark_repetitions=30 --benchmark_counters_tabular=true --benchmark_out_format=json --benchmark_enable_random_interleaving=true --benchmark_min_warmup_time=10 --benchmark_min_time=1000000x
+BENCHMARK_OPTIONS = --benchmark_repetitions=3 --benchmark_counters_tabular=true --benchmark_out_format=json
 RUN_MATLAB = matlab -nodisplay -nosplash -nodesktop -r
 BENCH_DATA = bench_result/data
 BENCH_IMAGE = bench_result/image
 PYTHON = python3
 
 # run-benchmark-blaze-dynamic:
-# 	$(BENCH_TMPC) --benchmark_filter="BM_gemm_blaze_dynamic<double>*" $(BENCHMARK_OPTIONS) \
+# 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_blaze_dynamic<double>*" $(BENCHMARK_OPTIONS) \
 # 		--benchmark_out=${BENCH_DATA}/blaze-dynamic.json
 
-${BENCH_DATA}/loop-naive.json:
-	$(BENCH_TMPC) --benchmark_filter="BM_gemm_loop_naive<double>*" $(BENCHMARK_OPTIONS) \
-		--benchmark_out=${BENCH_DATA}/loop-naive.json
+# ${BENCH_DATA}/loop-naive.json:
+# 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_loop_naive<double>*" $(BENCHMARK_OPTIONS) \
+# 		--benchmark_out=${BENCH_DATA}/loop-naive.json
 
 #
 # DGEMM/SGEMM bechmarks data
@@ -65,18 +65,22 @@ ${BENCH_DATA}/sgemm-blaze-static.json: $(BENCH_BLAZE)
 ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<double, .+>" $(BENCHMARK_OPTIONS) \
 		--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-panel.json
+	cp $@ ${BENCH_DATA}
 
 ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_panel<double>" $(BENCHMARK_OPTIONS) \
 		--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-panel.json
+	cp $@ ${BENCH_DATA}
 
 ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_plain<double, .+>" $(BENCHMARK_OPTIONS) \
 		--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-static-plain.json
+	cp $@ ${BENCH_DATA}
 
 ${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_plain<double>" $(BENCHMARK_OPTIONS) \
 		--benchmark_out=${BENCH_DATA}/${BENCH_BLAST_OUTPUT_DIR}/dgemm-blast-dynamic-plain.json
+	cp $@ ${BENCH_DATA}
 
 ${BENCH_DATA}/sgemm-blast-static-panel.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<float, .+>" $(BENCHMARK_OPTIONS) \

diff --git a/bench/analysis/dgemm_performance.py b/bench/analysis/dgemm_performance.py
@@ -2,8 +2,6 @@
 matplotlib.use("Agg")
 import matplotlib.pyplot as plt
 import json
-import glob
-import pathlib
 
 def filter_aggregate(benchmarks, name):
     result = []
@@ -16,30 +14,31 @@ def filter_aggregate(benchmarks, name):
     return result
 
 
+def load_benchmark(file_name):
+    with open(file_name) as f:
+        return filter_aggregate(json.load(f)['benchmarks'], 'median')
+
+
 factor = 1e+9 # Giga
 style = 'o-'
 line_width = 1
 
-# prop_cycle = plt.rcParams['axes.prop_cycle']
-# c = prop_cycle.by_key()['color']
-
 plots = [
-    # {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
-    # {'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
+    {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
+    {'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
     {'data_file': 'dgemm-blasfeo.json', 'label': 'BLASFEO'},
     # {'data_file': 'dgemm-blasfeo-blas.json', 'label': 'BLASFEO*'},
-    # {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
+    {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
     # {'data_file': 'dgemm-eigen-dynamic.json', 'label': 'Eigen (D)'},
-    # {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
+    {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
     # {'data_file': 'dgemm-blaze-dynamic.json', 'label': 'Blaze (D)'},
     # {'data_file': 'dgemm-blaze-static.json', 'label': 'Blaze (S)'},
+    {'data_file': 'dgemm-blast-static-panel.json', 'label': 'StaPan'},
+    {'data_file': 'dgemm-blast-static-plain.json', 'label': 'StaPla'},
+    {'data_file': 'dgemm-blast-dynamic-panel.json', 'label': 'DynPan'},
+    {'data_file': 'dgemm-blast-dynamic-plain.json', 'label': 'DynPla'},
 ]
 
-for benchmark_file, benchmark_label in [('dgemm-blast-static-panel.json', 'SP'), ('dgemm-blast-static-plain.json', 'SD'), ('dgemm-blast-dynamic-panel.json', 'DP'), ('dgemm-blast-dynamic-plain.json', 'DD')]:
-    files = glob.glob('./**/' + benchmark_file, recursive=True, root_dir='bench_result/data')
-    for file in files:
-        plots.append({'data_file': file, 'label': f'BLAST ({benchmark_label}) {pathlib.Path(file).parent.stem}'})
-
 fig = plt.figure(figsize=[10, 6])
 ax = fig.subplots()
 

diff --git a/bench/analysis/dgemm_performance_ratio.py b/bench/analysis/dgemm_performance_ratio.py
@@ -16,7 +16,7 @@ def filter_aggregate(benchmarks, name):
 
 def load_benchmark(file_name):
     with open(file_name) as f:
-        return filter_aggregate(json.load(f)['benchmarks'], 'mean')
+        return filter_aggregate(json.load(f)['benchmarks'], 'median')
 
 
 factor = 1e+9 # Giga

diff --git a/cmake/InitialCache.cmake b/cmake/InitialCache.cmake
@@ -1,6 +1,6 @@
-set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Build type (Debug, Release, etc.)")
-set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx -mavx2 -msse4 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3<avx2>'" CACHE STRING "Recommended compiler flags")
-set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build")
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type (Debug, Release, etc.)")
+set(CMAKE_CXX_FLAGS "-march=native -mfma -mavx2 -fno-math-errno -DXSIMD_DEFAULT_ARCH='fma3<avx2>'" CACHE STRING "Recommended compiler flags")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG -ffast-math" CACHE STRING "Recommended compiler flags for a Release build")
 set(BLAST_WITH_TEST ON CACHE BOOL "Build blast tests")
 set(BLAST_WITH_BENCHMARK ON CACHE BOOL "Build blast benchmarks")
 set(BLAST_WITH_BLASFEO ON CACHE BOOL "Build blasfeo C++ interface")

diff --git a/docker/x86_64/Dockerfile b/docker/x86_64/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:kinetic
+FROM ubuntu:noble
 WORKDIR /root
 RUN apt-get update
 RUN apt-get upgrade -y