From 84c8c583aedb97ea9813355539cb11e980ed6f2e Mon Sep 17 00:00:00 2001
From: Robin Verschueren <robin.verschueren@pm.me>
Date: Tue, 6 Aug 2024 11:07:37 +0200
Subject: [PATCH] Compare performance across Blast commits

---
 Makefile                            | 25 +++++++++++++------------
 bench/analysis/dgemm_performance.py | 20 +++++++++++---------
 2 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/Makefile b/Makefile
index a1fa3930..3240a33c 100644
--- a/Makefile
+++ b/Makefile
@@ -61,21 +61,21 @@ ${BENCH_DATA}/sgemm-blaze-static.json: $(BENCH_BLAZE)
 	$(BENCH_BLAZE) --benchmark_filter="BM_gemm_static<float>*" $(BENCHMARK_OPTIONS) \
 		--benchmark_out=${BENCH_DATA}/sgemm-blaze-static.json
 
-${BENCH_DATA}/dgemm-blast-static-panel.json: $(BENCH_BLAST)
+${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-static-panel.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<double, .+>" $(BENCHMARK_OPTIONS) \
-		--benchmark_out=${BENCH_DATA}/dgemm-blast-static-panel.json
+		--benchmark_out=${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-static-panel.json
 
-${BENCH_DATA}/dgemm-blast-dynamic-panel.json: $(BENCH_BLAST)
+${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-dynamic-panel.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_panel<double>" $(BENCHMARK_OPTIONS) \
-		--benchmark_out=${BENCH_DATA}/dgemm-blast-dynamic-panel.json
+		--benchmark_out=${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-dynamic-panel.json
 
-${BENCH_DATA}/dgemm-blast-static-plain.json: $(BENCH_BLAST)
+${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-static-plain.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_plain<double, .+>" $(BENCHMARK_OPTIONS) \
-		--benchmark_out=${BENCH_DATA}/dgemm-blast-static-plain.json
+		--benchmark_out=${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-static-plain.json
 
-${BENCH_DATA}/dgemm-blast-dynamic-plain.json: $(BENCH_BLAST)
+${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-dynamic-plain.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_dynamic_plain<double>" $(BENCHMARK_OPTIONS) \
-		--benchmark_out=${BENCH_DATA}/dgemm-blast-dynamic-plain.json
+		--benchmark_out=${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-dynamic-plain.json
 
 ${BENCH_DATA}/sgemm-blast-static-panel.json: $(BENCH_BLAST)
 	$(BENCH_BLAST) --benchmark_filter="BM_gemm_static_panel<float, .+>" $(BENCHMARK_OPTIONS) \
@@ -94,16 +94,17 @@ ${BENCH_DATA}/sgemm-libxsmm.json: $(BENCH_LIBXSMM)
 		--benchmark_out=${BENCH_DATA}/sgemm-libxsmm.json
 
 dgemm-benchmarks: \
+	$(shell mkdir -p ${BENCH_DATA}/$(shell git rev-parse --short HEAD)) \
 	${BENCH_DATA}/dgemm-openblas.json \
 	${BENCH_DATA}/dgemm-mkl.json \
 	${BENCH_DATA}/dgemm-libxsmm.json \
 	${BENCH_DATA}/dgemm-blasfeo.json \
 	${BENCH_DATA}/dgemm-blaze-static.json \
 	${BENCH_DATA}/dgemm-eigen-static.json \
-	${BENCH_DATA}/dgemm-blast-static-panel.json \
-	${BENCH_DATA}/dgemm-blast-dynamic-panel.json \
-	${BENCH_DATA}/dgemm-blast-static-plain.json \
-	${BENCH_DATA}/dgemm-blast-dynamic-plain.json
+	${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-static-panel.json \
+	${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-dynamic-panel.json \
+	${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-static-plain.json \
+	${BENCH_DATA}/$(shell git rev-parse --short HEAD)/dgemm-blast-dynamic-plain.json
 
 
 #
diff --git a/bench/analysis/dgemm_performance.py b/bench/analysis/dgemm_performance.py
index 7f401e91..b3c16b40 100644
--- a/bench/analysis/dgemm_performance.py
+++ b/bench/analysis/dgemm_performance.py
@@ -1,6 +1,7 @@
 import matplotlib.pyplot as plt
 import json
-
+import glob
+import pathlib
 
 def filter_aggregate(benchmarks, name):
     result = []
@@ -22,20 +23,21 @@ def filter_aggregate(benchmarks, name):
 
 plots = [
     # {'data_file': 'dgemm-openblas.json', 'label': 'OpenBLAS'},
-    {'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
-    {'data_file': 'dgemm-blasfeo.json', 'label': 'BLASFEO'},
+    # {'data_file': 'dgemm-mkl.json', 'label': 'MKL'},
+    # {'data_file': 'dgemm-blasfeo.json', 'label': 'BLASFEO'},
     # {'data_file': 'dgemm-blasfeo-blas.json', 'label': 'BLASFEO*'},
-    {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
+    # {'data_file': 'dgemm-libxsmm.json', 'label': 'LIBXSMM'},
     # {'data_file': 'dgemm-eigen-dynamic.json', 'label': 'Eigen (D)'},
     # {'data_file': 'dgemm-eigen-static.json', 'label': 'Eigen (S)'},
     # {'data_file': 'dgemm-blaze-dynamic.json', 'label': 'Blaze (D)'},
-    {'data_file': 'dgemm-blaze-static.json', 'label': 'Blaze (S)'},
-    {'data_file': 'dgemm-blast-static-panel.json', 'label': 'BLAST (SP)'},
-    {'data_file': 'dgemm-blast-static-plain.json', 'label': 'BLAST (SD)'},
-    {'data_file': 'dgemm-blast-dynamic-panel.json', 'label': 'BLAST (DP)'},
-    {'data_file': 'dgemm-blast-dynamic-plain.json', 'label': 'BLAST (DD)'},
+    # {'data_file': 'dgemm-blaze-static.json', 'label': 'Blaze (S)'},
 ]
 
+for benchmark_file, benchmark_label in [('dgemm-blast-static-panel.json', 'SP'), ('dgemm-blast-static-plain.json', 'SD'), ('dgemm-blast-dynamic-panel.json', 'DP'), ('dgemm-blast-dynamic-plain.json', 'DD')]:
+    files = glob.glob('./**/' + benchmark_file, recursive=True, root_dir='bench_result/data')
+    for file in files:
+        plots.append({'data_file': file, 'label': f'BLAST ({benchmark_label}) {pathlib.Path(file).parent.stem}'})
+
 fig = plt.figure(figsize=[10, 6])
 ax = fig.subplots()