From 48961d7359d074a2df888ecab02e233891ad1fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20M=C3=BCller?= Date: Wed, 25 Nov 2020 16:54:44 +0100 Subject: [PATCH 1/2] Add performance test for mock backend [WIP] --- test/system/test_speed_mock.py | 207 +++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 test/system/test_speed_mock.py diff --git a/test/system/test_speed_mock.py b/test/system/test_speed_mock.py new file mode 100644 index 000000000..e41e924e0 --- /dev/null +++ b/test/system/test_speed_mock.py @@ -0,0 +1,207 @@ +try: + import unittest2 as unittest +except ImportError: + import unittest + +# import pyNN.nest as sim +import pyNN.mock as sim + +from pyNN.utility import Timer + + +class PopulationTest(unittest.TestCase): + @staticmethod + def do_scaling_per_population_test(N): + timer = Timer() + timer.start() + sim.setup() + timer.mark("setup") + p = sim.Population(N, sim.IF_curr_exp()) + timer.mark("Population: " + str(N)) + sim.end() + timer.mark("end") + elapsed_time = timer.elapsed_time() + relative_elapsed_time = elapsed_time / N + print( + "Creating a {}-sized population took {}s ({}s per neuron)".format( + N, elapsed_time, relative_elapsed_time + ) + ) + + def test_scaling_per_population(self, sim=sim): + for powerN in range(13): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_per_population_test(N) + + @staticmethod + def do_scaling_test(N): + timer = Timer() + timer.start() + sim.setup() + timer.mark("setup") + for _ in range(N): + p = sim.Population(1, sim.IF_curr_exp()) + timer.mark("Population: " + str(N)) + sim.end() + timer.mark("end") + elapsed_time = timer.elapsed_time() + relative_elapsed_time = elapsed_time / N + print( + "Creating {} populations took {}s ({}s per population)".format( + N, elapsed_time, relative_elapsed_time + ) + ) + + def test_scaling(self, sim=sim): + for powerN in range(13): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_test(N) + + @staticmethod + def _add_dummy_parameters(celltype, M): + for i in range(M): + pname = "tmp{}".format(i) + celltype.default_parameters[pname] = 0.0 + celltype.units[pname] = "mV" + celltype.translations[pname] = { + "translated_name": pname.upper(), + "forward_transform": pname, + "reverse_transform": pname.upper(), + } + + @staticmethod + def _remove_dummy_parameters(celltype, M): + for i in range(M): + pname = "tmp{}".format(i) + del celltype.default_parameters[pname] + del celltype.units[pname] + del celltype.translations[pname] + + @staticmethod + def do_scaling_cellparams_test(N, M): + # copy.deepcopy doesn't help here => we add and restore manually + celltype = sim.IF_cond_exp + assert len(celltype.get_parameter_names()) < 100 + PopulationTest._add_dummy_parameters(celltype, M) + sim.setup() + t0 = Timer() + t0.start() + pop = sim.Population(N, celltype()) # this is the culprit + elapsed_time = t0.elapsed_time() + relative_elapsed_time = elapsed_time / M + print( + "Creating a population with {} parameters took {} ({} per parameter)".format( + len(celltype.get_parameter_names()), elapsed_time, relative_elapsed_time + ) + ) + sim.end() + PopulationTest._remove_dummy_parameters(celltype, M) + + def test_scaling_cellparams(self, sim=sim): + for powerN in range(16): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_cellparams_test(1, N) + + @staticmethod + def do_scaling_cellparams_popview_test(N, M): + # copy.deepcopy doesn't help here => we add and restore manually + celltype = sim.IF_cond_exp + assert len(celltype.get_parameter_names()) < 100 + PopulationTest._add_dummy_parameters(celltype, M) + sim.setup() + pop = sim.Population(N, celltype()) + pview = sim.PopulationView(pop, [0]) + post_cell = sim.simulator.ID(pview.first_id) + post_cell.parent = pop # this is the culprit + t0 = Timer() + t0.start() + post_index = pview.id_to_index(post_cell) + elapsed_time = t0.elapsed_time() + relative_elapsed_time = elapsed_time / M + print( + "Calling id_to_index on a view into a population with {} parameters took {} ({} per parameter)".format( + len(celltype.get_parameter_names()), elapsed_time, relative_elapsed_time + ) + ) + sim.end() + PopulationTest._remove_dummy_parameters(celltype, M) + + def test_scaling_cellparams_popview(self, sim=sim): + for powerN in range(8): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_cellparams_popview_test(1, N) + + +class ProjectionTest(unittest.TestCase): + @staticmethod + def do_scaling_per_projection_test(N): + sim.setup() + pre = sim.Population(N, sim.IF_cond_exp()) + post = sim.Population(N, sim.IF_cond_exp()) + + timer = Timer() + timer.start() + proj = sim.Projection( + pre, post, sim.OneToOneConnector(), synapse_type=sim.StaticSynapse(weight=1.0) + ) + timer.mark("Projection: " + str(N)) + elapsed_time = timer.elapsed_time() + relative_elapsed_time = elapsed_time / N + + sim.end() + print( + "Creating {}-sized projection took {}s ({}s per synapse)".format( + N, elapsed_time, relative_elapsed_time + ) + ) + + def test_scaling_per_projection(self, sim=sim): + for powerN in range(13): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_per_projection_test(N) + + @staticmethod + def do_scaling_test(N): + sim.setup() + pre = sim.Population(N, sim.IF_cond_exp()) + post = sim.Population(N, sim.IF_cond_exp()) + + timer = Timer() + timer.start() + for i in range(N): + + # FIXME: add check for class-vs-instance in connector, it fails if the connector is a class + proj = sim.Projection( + pre[i : i + 1], + post[i : i + 1], + sim.OneToOneConnector(), + synapse_type=sim.StaticSynapse(weight=1.0), + ) + timer.mark("Projection: " + str(N)) + elapsed_time = timer.elapsed_time() + relative_elapsed_time = elapsed_time / N + + sim.end() + + print( + "Creating {} projections took {}s ({}s per projection)".format( + N, elapsed_time, relative_elapsed_time + ) + ) + + def test_scaling(self, sim=sim): + for powerN in range(13): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_test(N) + + +# import profile +if __name__ == "__main__": + unittest.main() + # profile.run('print(PopulationTest.do_scaling_cellparams_popview_test(1, 100))') From a58cc5120ab6b7475bac3d8e696a37636219bb30 Mon Sep 17 00:00:00 2001 From: Christian Mauch Date: Thu, 26 Nov 2020 14:01:50 +0100 Subject: [PATCH 2/2] Update speed test * cleanup * expand id_to_index test case --- test/system/test_speed_mock.py | 128 ++++++++++++++++----------------- 1 file changed, 63 insertions(+), 65 deletions(-) diff --git a/test/system/test_speed_mock.py b/test/system/test_speed_mock.py index e41e924e0..d475ae3d5 100644 --- a/test/system/test_speed_mock.py +++ b/test/system/test_speed_mock.py @@ -3,9 +3,8 @@ except ImportError: import unittest -# import pyNN.nest as sim import pyNN.mock as sim - +import numpy as np from pyNN.utility import Timer @@ -16,7 +15,7 @@ def do_scaling_per_population_test(N): timer.start() sim.setup() timer.mark("setup") - p = sim.Population(N, sim.IF_curr_exp()) + sim.Population(N, sim.IF_curr_exp()) timer.mark("Population: " + str(N)) sim.end() timer.mark("end") @@ -24,9 +23,7 @@ def do_scaling_per_population_test(N): relative_elapsed_time = elapsed_time / N print( "Creating a {}-sized population took {}s ({}s per neuron)".format( - N, elapsed_time, relative_elapsed_time - ) - ) + N, elapsed_time, relative_elapsed_time)) def test_scaling_per_population(self, sim=sim): for powerN in range(13): @@ -34,6 +31,29 @@ def test_scaling_per_population(self, sim=sim): with self.subTest(N=N): self.do_scaling_per_population_test(N) + @staticmethod + def do_scaling_per_population_view_test(N, M): + sim.setup() + pop = sim.Population(2**M, sim.IF_curr_exp()) + timer = Timer() + timer.start() + sim.PopulationView(pop, np.array(range(N))) + timer.mark("PopulationView size: " + str(N)) + sim.end() + timer.mark("end") + elapsed_time = timer.elapsed_time() + relative_elapsed_time = elapsed_time / N + print("Creating a {}-sized population view took {}s " + "({}s per view)".format( + N, elapsed_time, relative_elapsed_time)) + + def test_scaling_per_population_view(self, sim=sim): + M = 13 + for powerN in range(M): + N = 2 ** powerN + with self.subTest(N=N): + self.do_scaling_per_population_view_test(N, M) + @staticmethod def do_scaling_test(N): timer = Timer() @@ -41,7 +61,7 @@ def do_scaling_test(N): sim.setup() timer.mark("setup") for _ in range(N): - p = sim.Population(1, sim.IF_curr_exp()) + sim.Population(1, sim.IF_curr_exp()) timer.mark("Population: " + str(N)) sim.end() timer.mark("end") @@ -49,9 +69,7 @@ def do_scaling_test(N): relative_elapsed_time = elapsed_time / N print( "Creating {} populations took {}s ({}s per population)".format( - N, elapsed_time, relative_elapsed_time - ) - ) + N, elapsed_time, relative_elapsed_time)) def test_scaling(self, sim=sim): for powerN in range(13): @@ -80,60 +98,45 @@ def _remove_dummy_parameters(celltype, M): del celltype.translations[pname] @staticmethod - def do_scaling_cellparams_test(N, M): + def do_scaling_cellparams_id_to_index_test(N): # copy.deepcopy doesn't help here => we add and restore manually celltype = sim.IF_cond_exp assert len(celltype.get_parameter_names()) < 100 - PopulationTest._add_dummy_parameters(celltype, M) + PopulationTest._add_dummy_parameters(celltype, N) + num_params = len(celltype.get_parameter_names()) sim.setup() - t0 = Timer() - t0.start() - pop = sim.Population(N, celltype()) # this is the culprit - elapsed_time = t0.elapsed_time() - relative_elapsed_time = elapsed_time / M - print( - "Creating a population with {} parameters took {} ({} per parameter)".format( - len(celltype.get_parameter_names()), elapsed_time, relative_elapsed_time - ) - ) - sim.end() - PopulationTest._remove_dummy_parameters(celltype, M) - - def test_scaling_cellparams(self, sim=sim): - for powerN in range(16): - N = 2 ** powerN - with self.subTest(N=N): - self.do_scaling_cellparams_test(1, N) - - @staticmethod - def do_scaling_cellparams_popview_test(N, M): - # copy.deepcopy doesn't help here => we add and restore manually - celltype = sim.IF_cond_exp - assert len(celltype.get_parameter_names()) < 100 - PopulationTest._add_dummy_parameters(celltype, M) - sim.setup() - pop = sim.Population(N, celltype()) + pop = sim.Population(1, celltype()) pview = sim.PopulationView(pop, [0]) + # we specifically create an ID to check that having a parent leads to + # the extreme slow down of id_to_index post_cell = sim.simulator.ID(pview.first_id) post_cell.parent = pop # this is the culprit t0 = Timer() t0.start() - post_index = pview.id_to_index(post_cell) + pop.id_to_index(post_cell) elapsed_time = t0.elapsed_time() - relative_elapsed_time = elapsed_time / M - print( - "Calling id_to_index on a view into a population with {} parameters took {} ({} per parameter)".format( - len(celltype.get_parameter_names()), elapsed_time, relative_elapsed_time - ) - ) + relative_elapsed_time = elapsed_time / num_params + print("Calling id_to_index on a population with {} " + "parameters took {} ({} per parameter)".format( + num_params, + elapsed_time, relative_elapsed_time)) + t1 = Timer() + t1.start() + pview.id_to_index(post_cell) + elapsed_time = t1.elapsed_time() + relative_elapsed_time = elapsed_time / num_params + print("Calling id_to_index on a view into a population with {} " + "parameters took {} ({} per parameter)".format( + num_params, + elapsed_time, relative_elapsed_time)) sim.end() - PopulationTest._remove_dummy_parameters(celltype, M) + PopulationTest._remove_dummy_parameters(celltype, N) - def test_scaling_cellparams_popview(self, sim=sim): + def test_scaling_cellparams_id_to_index(self, sim=sim): for powerN in range(8): N = 2 ** powerN with self.subTest(N=N): - self.do_scaling_cellparams_popview_test(1, N) + self.do_scaling_cellparams_id_to_index_test(N) class ProjectionTest(unittest.TestCase): @@ -145,8 +148,11 @@ def do_scaling_per_projection_test(N): timer = Timer() timer.start() - proj = sim.Projection( - pre, post, sim.OneToOneConnector(), synapse_type=sim.StaticSynapse(weight=1.0) + sim.Projection( + pre, + post, + sim.OneToOneConnector(), + synapse_type=sim.StaticSynapse(weight=1.0) ) timer.mark("Projection: " + str(N)) elapsed_time = timer.elapsed_time() @@ -155,9 +161,7 @@ def do_scaling_per_projection_test(N): sim.end() print( "Creating {}-sized projection took {}s ({}s per synapse)".format( - N, elapsed_time, relative_elapsed_time - ) - ) + N, elapsed_time, relative_elapsed_time)) def test_scaling_per_projection(self, sim=sim): for powerN in range(13): @@ -174,13 +178,11 @@ def do_scaling_test(N): timer = Timer() timer.start() for i in range(N): - - # FIXME: add check for class-vs-instance in connector, it fails if the connector is a class - proj = sim.Projection( - pre[i : i + 1], - post[i : i + 1], + sim.Projection( + pre[i: i + 1], + post[i: i + 1], sim.OneToOneConnector(), - synapse_type=sim.StaticSynapse(weight=1.0), + synapse_type=sim.StaticSynapse(weight=1.0) ) timer.mark("Projection: " + str(N)) elapsed_time = timer.elapsed_time() @@ -190,9 +192,7 @@ def do_scaling_test(N): print( "Creating {} projections took {}s ({}s per projection)".format( - N, elapsed_time, relative_elapsed_time - ) - ) + N, elapsed_time, relative_elapsed_time)) def test_scaling(self, sim=sim): for powerN in range(13): @@ -201,7 +201,5 @@ def test_scaling(self, sim=sim): self.do_scaling_test(N) -# import profile if __name__ == "__main__": unittest.main() - # profile.run('print(PopulationTest.do_scaling_cellparams_popview_test(1, 100))')