From 946b24732bac543a580830de4ab35110148462cf Mon Sep 17 00:00:00 2001 From: Bryce Drennan Date: Tue, 9 Apr 2024 17:23:52 -0700 Subject: [PATCH] feature: add the option `--randomly-seed-per-test` to use a different seed for each test --- CHANGELOG.rst | 4 +++ README.rst | 3 +++ src/pytest_randomly/__init__.py | 19 ++++++++++++- tests/test_pytest_randomly.py | 48 ++++++++++++++++++++++++++------- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 117e439..1d2bd2b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ Changelog ========= +* Add the option ``--randomly-seed-per-test`` to use a different seed for each test. + + Resolves `Issue #600 `__ + 3.15.0 (2023-08-15) ------------------- diff --git a/README.rst b/README.rst index 6d6c968..d16b6da 100644 --- a/README.rst +++ b/README.rst @@ -160,6 +160,9 @@ You can disable behaviours you don't like with the following flags: the start of every test * ``--randomly-dont-reorganize`` - turn off the shuffling of the order of tests +By default each test starts out with the same seed, if you'd like a different one +per test you can use the ``--randomly-seed-per-test`` flag. + The plugin appears to Pytest with the name 'randomly'. To disable it altogether, you can use the ``-p`` argument, for example: diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index f2cb12c..6d5c226 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -111,6 +111,15 @@ def pytest_addoption(parser: Parser) -> None: default=True, help="Stop pytest-randomly from randomly reorganizing the test order.", ) + group._addoption( + "--randomly-dont-seed-per-test", + action="store_false", + dest="randomly_seed_per_test", + default=True, + help="""Use a different seed for each test. Can be helpful for getting + different random data in each test, but still having reproducible + tests. Default behaviour: False.""", + ) def pytest_configure(config: Config) -> None: @@ -209,9 +218,17 @@ def pytest_runtest_setup(item: Item) -> None: _reseed(item.config, -1) +def seed_from_string(string: str) -> int: + return int(hashlib.md5(string.encode()).hexdigest(), 16) + + def pytest_runtest_call(item: Item) -> None: if item.config.getoption("randomly_reset_seed"): - _reseed(item.config) + if item.config.getoption("randomly_seed_per_test"): + test_offset = seed_from_string(item.nodeid) + 100 + else: + test_offset = 0 + _reseed(item.config, offset=test_offset) def pytest_runtest_teardown(item: Item) -> None: diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index 9449aca..9d46f33 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -78,7 +78,33 @@ def test_b(): assert test_b.num == test_a.num """ ) - out = ourtester.runpytest("--randomly-dont-reorganize") + out = ourtester.runpytest( + "--randomly-dont-reorganize", "--randomly-dont-seed-per-test" + ) + out.assert_outcomes(passed=2, failed=0) + + +def test_it_uses_different_random_seed_per_test(ourtester): + """ + Run a pair of tests that generate a number and assert they produce + different numbers. + """ + ourtester.makepyfile( + test_one=""" + import random + + def test_a(): + test_a.num = random.random() + if hasattr(test_b, 'num'): + assert test_a.num != test_b.num + + def test_b(): + test_b.num = random.random() + if hasattr(test_a, 'num'): + assert test_b.num != test_a.num + """ + ) + out = ourtester.runpytest() out.assert_outcomes(passed=2, failed=0) @@ -601,7 +627,7 @@ def test_two(myfixture): assert random.getstate() == state_at_seed_two """ ) - args = ["--randomly-seed=2"] + args = ["--randomly-seed=2", "--randomly-dont-seed-per-test"] out = ourtester.runpytest(*args) out.assert_outcomes(passed=2) @@ -633,7 +659,7 @@ def test_b(): """ ) - out = ourtester.runpytest("--randomly-seed=1") + out = ourtester.runpytest("--randomly-seed=1", "--randomly-dont-seed-per-test") out.assert_outcomes(passed=2) @@ -645,10 +671,10 @@ def test_faker(ourtester): fake = Faker() def test_one(): - assert fake.name() == 'Ryan Gallagher' + assert fake.name() == 'Justin Richard' def test_two(): - assert fake.name() == 'Ryan Gallagher' + assert fake.name() == 'Tiffany Williams' """ ) @@ -692,7 +718,7 @@ def test_b(): """ ) - out = ourtester.runpytest("--randomly-seed=1") + out = ourtester.runpytest("--randomly-seed=1", "--randomly-dont-seed-per-test") out.assert_outcomes(passed=2) @@ -702,10 +728,10 @@ def test_numpy(ourtester): import numpy as np def test_one(): - assert np.random.rand() == 0.417022004702574 + assert np.random.rand() == 0.46479378116435255 def test_two(): - assert np.random.rand() == 0.417022004702574 + assert np.random.rand() == 0.6413112443155088 """ ) @@ -765,7 +791,7 @@ def fake_entry_points(*, group): entry_points.append(_FakeEntryPoint("test_seeder", reseed)) # Need to run in-process so that monkeypatching works - pytester.runpytest_inprocess("--randomly-seed=1") + pytester.runpytest_inprocess("--randomly-seed=1", "--randomly-dont-seed-per-test") assert reseed.mock_calls == [ mock.call(1), mock.call(1), @@ -775,7 +801,9 @@ def fake_entry_points(*, group): ] reseed.mock_calls[:] = [] - pytester.runpytest_inprocess("--randomly-seed=424242") + pytester.runpytest_inprocess( + "--randomly-seed=424242", "--randomly-dont-seed-per-test" + ) assert reseed.mock_calls == [ mock.call(424242), mock.call(424242),