From 7e3e67c300648b7d936bc6dd2ea675062a34e5da Mon Sep 17 00:00:00 2001 From: Alexander Hess Date: Tue, 15 Oct 2024 01:49:32 +0200 Subject: [PATCH] Add smoke tests - extend `pytest` with an option to run only the minimum number of (unit) test cases to just keep the coverage at 100% - rationale: + many of the unit test cases partly overlap with respect to the lines of source code executed + also, integration tests, by definition, do not contribute to a higher test coverage - implementation: mark "redundant" test cases as one of: + `pytest.mark.integration_test` => code usage from the perspective of the end user + `pytest.mark.overlapping_test` => tests not contributing to the 100% coverage + `pytest.mark.sanity_test` => tests providing confidence in the test data - add `tests.conftest` module => programatically convert the above markers into `@pytest.mark.no_cover` and collect the non-"redundant" tests - add nox session "test-fast" to run only the minimum number of (unit) test while holding coverage at 100% - refactor some test modules + wrap some test cases in a class + move sanity tests to the end of the files --- noxfile.py | 20 ++++++++++ tests/conftest.py | 59 +++++++++++++++++++++++++++++ tests/elements/test_galois.py | 40 +++++++++++++++++--- tests/fields/test_axioms.py | 4 ++ tests/fields/test_base.py | 14 +++++++ tests/fields/test_complex.py | 4 ++ tests/fields/test_galois.py | 4 ++ tests/fields/test_rational.py | 4 ++ tests/test_docstrings.py | 1 + tests/test_top_level_imports.py | 1 + tests/test_version.py | 66 ++++++++++++++++++--------------- 11 files changed, 182 insertions(+), 35 deletions(-) create mode 100644 tests/conftest.py diff --git a/noxfile.py b/noxfile.py index fc0fee2..e03e1bb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -254,6 +254,14 @@ def test(session: nox.Session) -> None: args = posargs or ( "--cov", "--no-cov-on-fail", + *( # If this function is run via the "test-fast" session, + ( # the following arguments are added: + "--cov-fail-under=100", + "--smoke-tests-only", + ) + if session.env.get("_smoke_tests_only") + else () + ), TEST_RANDOM_SEED, TESTS_LOCATION, ) @@ -339,6 +347,18 @@ def test_docstrings(session: nox.Session) -> None: session.run("xdoctest", "src/lalib") +@nox.session(name="test-fast", python=MAIN_PYTHON) +def test_fast(session: nox.Session) -> None: + """Test code with `pytest`, selected (smoke) tests only. + + The (unit) test cases are selected such that their number + is minimal but the achieved coverage remains at 100%. + """ + # See implementation notes in `pre_commit_test_hook()` below + session.env["_smoke_tests_only"] = "true" + test(session) + + @nox.session(name="clean-cwd", python=MAIN_PYTHON, venv_backend="none") def clean_cwd(session: nox.Session) -> None: """Remove (almost) all glob patterns listed in git's ignore file. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7917b02 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,59 @@ +"""Configurations and utilities for all tests.""" + +import pytest + + +def pytest_addoption(parser): + """Define custom CLI options for `pytest`.""" + parser.addoption( + "--smoke-tests-only", + action="store_true", + default=False, + help="Run the minimum number of (unit) tests to achieve full coverage", + ) + + +def pytest_configure(config): + """Define custom markers explicitly.""" + config.addinivalue_line( + "markers", + "integration_test: non-unit test case; skipped during coverage reporting", + ) + config.addinivalue_line( + "markers", + "overlapping_test: test case not contributing towards higher coverage", + ) + config.addinivalue_line( + "markers", + "sanity_test: test case providing confidence in the test data", + ) + + +def pytest_collection_modifyitems(config, items): + """Pre-process the test cases programatically. + + - Add `no_cover` marker to test cases with any of these markers: + + "integration_test" + + "overlapping_test" + + "sanity_test" + - Select test cases with none of the above markers as smoke tests, + i.e., the minimum number of test cases to achieve 100% coverage + """ + smoke_tests = [] + + for item in items: + if ( + "integration_test" in item.keywords + or "overlapping_test" in item.keywords + or "sanity_test" in item.keywords + ): + item.add_marker(pytest.mark.no_cover) + + elif config.getoption("--smoke-tests-only"): + smoke_tests.append(item) + + if config.getoption("--smoke-tests-only"): + if not smoke_tests: + pytest.exit("No smoke tests found") + + items[:] = smoke_tests diff --git a/tests/elements/test_galois.py b/tests/elements/test_galois.py index 06a4800..4dff652 100644 --- a/tests/elements/test_galois.py +++ b/tests/elements/test_galois.py @@ -78,11 +78,7 @@ zero_like_values = ( ) -def test_thresholds(): - """Sanity check for the thresholds used in the tests below.""" - assert utils.WITHIN_THRESHOLD < utils.DEFAULT_THRESHOLD < utils.NOT_WITHIN_THRESHOLD - - +@pytest.mark.overlapping_test class TestGF2SubClasses: """Test the sub-classes behind `one` and `zero`.""" @@ -122,6 +118,7 @@ class TestGF2Casting: result = cls(value, strict=False) assert result is one + @pytest.mark.overlapping_test @pytest.mark.parametrize("value", non_strict_one_like_values) def test_cannot_cast_ones_strictly(self, cls, value): """`gf2(value, strict=False)` returns `1`.""" @@ -156,6 +153,7 @@ class TestGF2Casting: with pytest.raises(ValueError, match="`1`-like or `0`-like"): cls(value, strict=strict) + @pytest.mark.overlapping_test @pytest.mark.parametrize("value", ["abc", (1,), [1]]) @pytest.mark.parametrize("strict", [True, False]) def test_cannot_cast_from_wrong_type(self, cls, value, strict): @@ -163,12 +161,14 @@ class TestGF2Casting: with pytest.raises(TypeError): cls(value, strict=strict) + @pytest.mark.overlapping_test @pytest.mark.parametrize("strict", [True, False]) def test_cannot_cast_from_nan_value(self, cls, strict): """Cannot create `one` or `zero` from undefined value.""" with pytest.raises(ValueError, match="`1`-like or `0`-like"): cls(float("NaN"), strict=strict) + @pytest.mark.overlapping_test @pytest.mark.parametrize("scaler", [1, 10, 100, 1000]) def test_get_one_if_within_threshold(self, cls, scaler): """`gf2()` returns `one` if `value` is larger than `threshold`.""" @@ -180,6 +180,7 @@ class TestGF2Casting: result = cls(value, strict=False, threshold=threshold) assert result is one + @pytest.mark.overlapping_test @pytest.mark.parametrize("scaler", [1, 10, 100, 1000]) @pytest.mark.parametrize("strict", [True, False]) def test_get_zero_if_within_threshold(self, cls, scaler, strict): @@ -192,6 +193,7 @@ class TestGF2Casting: assert result is zero +@pytest.mark.overlapping_test @pytest.mark.parametrize("strict", [True, False]) class TestGF2ConstructorWithoutCastedValue: """Test the `gf2` class's constructor. @@ -220,6 +222,7 @@ class TestGenericBehavior: with pytest.raises(RuntimeError, match="internal error"): gf2() + @pytest.mark.overlapping_test @pytest.mark.parametrize("cls", [gf2, GF2One, GF2Zero]) def test_create_singletons(self, cls): """Singleton pattern: The classes always return the same instance.""" @@ -227,6 +230,7 @@ class TestGenericBehavior: second = cls() assert first is second + @pytest.mark.overlapping_test @pytest.mark.parametrize("obj", [one, zero]) def test_sub_classes_return_objs(self, obj): """`type(one)` and `type(zero)` return ... @@ -239,6 +243,7 @@ class TestGenericBehavior: new_obj = sub_cls() assert new_obj is obj + @pytest.mark.overlapping_test @pytest.mark.parametrize("obj", [one, zero]) @pytest.mark.parametrize( "type_", @@ -253,6 +258,7 @@ class TestGenericBehavior: """`one` and `zero` are officially `Numbers`s.""" assert isinstance(obj, type_) + @pytest.mark.overlapping_test @pytest.mark.parametrize("cls", [gf2, GF2One, GF2Zero]) @pytest.mark.parametrize( "method", @@ -320,14 +326,17 @@ class TestGenericBehavior: class TestNumericBehavior: """Test how `one` and `zero` behave like numbers.""" + @pytest.mark.overlapping_test def test_make_complex(self): """`one` and `zero` behave like `1 + 0j` and `0 + 0j`.""" assert (complex(one), complex(zero)) == (1 + 0j, 0 + 0j) + @pytest.mark.overlapping_test def test_make_float(self): """`one` and `zero` behave like `1.0` and `0.0`.""" assert (float(one), float(zero)) == (1.0, 0.0) + @pytest.mark.overlapping_test @pytest.mark.parametrize("func", [int, hash]) def test_make_int(self, func): """`one` and `zero` behave like `1` and `0`. @@ -340,6 +349,7 @@ class TestNumericBehavior: """`one` and `zero` behave like `True` and `False`.""" assert (bool(one), bool(zero)) == (True, False) + @pytest.mark.overlapping_test @pytest.mark.parametrize("obj", [one, zero]) def test_get_abs_value(self, obj): """`abs(one)` and `abs(zero)` are `one` and `zero`.""" @@ -366,10 +376,12 @@ class TestNumericBehavior: """`one.conjugate()` and `zero.conjugate()` are `1 + 0j` and `0 + 0j`.""" assert (one.conjugate(), zero.conjugate()) == (1 + 0j, 0 + 0j) + @pytest.mark.overlapping_test def test_one_as_fraction(self): """`one.numerator / one.denominator` equals `1`.""" assert (one.numerator, one.denominator) == (1, 1) + @pytest.mark.overlapping_test def test_zero_as_fraction(self): """`one.numerator / one.denominator` equals `0`.""" assert (zero.numerator, zero.denominator) == (0, 1) @@ -378,11 +390,13 @@ class TestNumericBehavior: class TestComparison: """Test `one` and `zero` interact with relational operators.""" + @pytest.mark.overlapping_test @pytest.mark.parametrize("obj", [one, zero]) def test_equal_to_itself(self, obj): """`one` and `zero` are equal to themselves.""" assert obj == obj # noqa: PLR0124 + @pytest.mark.overlapping_test @pytest.mark.parametrize( ["first", "second"], [ @@ -401,6 +415,7 @@ class TestComparison: assert first == second assert second == first + @pytest.mark.overlapping_test @pytest.mark.parametrize( ["first", "second"], [ @@ -418,6 +433,7 @@ class TestComparison: assert first != second assert second != first + @pytest.mark.overlapping_test @pytest.mark.parametrize( ["first", "second"], [ @@ -441,27 +457,32 @@ class TestComparison: """`one > zero` and `one >= zero`.""" assert operator(one, zero) + @pytest.mark.overlapping_test @pytest.mark.parametrize("operator", [operator.lt, operator.le]) def test_one_not_smaller_than_or_equal_to_zero(self, operator): """`not one < zero` and `not one <= zero`.""" assert not operator(one, zero) + @pytest.mark.overlapping_test @pytest.mark.parametrize("operator", [operator.lt, operator.le]) def test_zero_smaller_than_or_equal_to_one(self, operator): """`zero < one` and `zero <= one`.""" assert operator(zero, one) + @pytest.mark.overlapping_test @pytest.mark.parametrize("operator", [operator.gt, operator.ge]) def test_zero_not_greater_than_or_equalt_to_one(self, operator): """`not zero > one` and `not zero >= one`.""" assert not operator(zero, one) + @pytest.mark.overlapping_test @pytest.mark.parametrize("obj", [one, zero]) def test_obj_not_strictly_greater_than_itself(self, obj): """`obj >= obj` but not `obj > obj`.""" assert obj >= obj # noqa: PLR0124 assert not obj > obj # noqa: PLR0124 + @pytest.mark.overlapping_test @pytest.mark.parametrize("obj", [one, zero]) def test_obj_not_strictly_smaller_than_itself(self, obj): """`obj <= obj` but not `obj < obj`.""" @@ -579,6 +600,7 @@ class TestArithmetic: result4 = gf2(int(abs(second)) * int(abs(first))) assert result4 is expected + @pytest.mark.overlapping_test @pytest.mark.parametrize( "objs", [ @@ -613,6 +635,7 @@ class TestArithmetic: result2 = gf2(operator(int(abs(first)), int(abs(second)))) assert result2 is expected + @pytest.mark.overlapping_test @pytest.mark.parametrize( "objs", [ @@ -762,6 +785,7 @@ class TestArithmetic: operator(42, obj) +@pytest.mark.overlapping_test @pytest.mark.skipif( not sys.version_info < (3, 11), reason='"typing-extensions" are installed to support Python 3.9 & 3.10', @@ -772,3 +796,9 @@ def test_can_import_typing_extensions(): importlib.reload(package) assert package.Self is not None + + +@pytest.mark.sanity_test +def test_thresholds(): + """Sanity check for the thresholds used in the tests below.""" + assert utils.WITHIN_THRESHOLD < utils.DEFAULT_THRESHOLD < utils.NOT_WITHIN_THRESHOLD diff --git a/tests/fields/test_axioms.py b/tests/fields/test_axioms.py index 67ddf70..680ebb6 100644 --- a/tests/fields/test_axioms.py +++ b/tests/fields/test_axioms.py @@ -11,6 +11,10 @@ import pytest from tests.fields import utils +# None of the test cases below contributes towards higher coverage +pytestmark = pytest.mark.integration_test + + @pytest.mark.repeat(utils.N_RANDOM_DRAWS) @pytest.mark.parametrize("field", utils.ALL_FIELDS) class TestAllFieldsManyTimes: diff --git a/tests/fields/test_base.py b/tests/fields/test_base.py index 31ac80f..80f00dc 100644 --- a/tests/fields/test_base.py +++ b/tests/fields/test_base.py @@ -39,6 +39,7 @@ class TestCastAndValidateFieldElements: an element of the `field`, and, if so, `.cast()` it as such. """ + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.NON_10_FIELDS) @pytest.mark.parametrize("value", utils.NUMBERS) def test_number_is_field_element(self, field, value): @@ -49,12 +50,14 @@ class TestCastAndValidateFieldElements: """ utils.is_field_element(field, value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) @pytest.mark.parametrize("value", utils.ONES_N_ZEROS) def test_one_and_zero_number_is_field_element(self, field, value): """`1`-like and `0`-like numbers are always `field` elements.""" utils.is_field_element(field, value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) @pytest.mark.parametrize("value", ["abc", (1, 2, 3)]) def test_non_numeric_value_is_not_field_element(self, field, value): @@ -93,6 +96,7 @@ class TestDTypes: """`field.dtype` must be a `type`.""" assert isinstance(field.dtype, type) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) def test_element_is_instance_of_field_dtype(self, field): """Elements are an instance of `field.dtype`.""" @@ -100,6 +104,7 @@ class TestDTypes: assert isinstance(element, field.dtype) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) def test_element_dtype_is_subclass_of_field_dtype(self, field): """Elements may have a more specific `.dtype` than their `field.dtype`.""" @@ -119,6 +124,7 @@ class TestIsZero: assert field.zero == value assert field.is_zero(value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) def test_is_almost_zero(self, field): """`value` is within an acceptable threshold of `field.zero`.""" @@ -127,6 +133,7 @@ class TestIsZero: assert pytest.approx(field.zero, abs=utils.DEFAULT_THRESHOLD) == value assert field.is_zero(value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.NON_10_FIELDS) def test_is_slightly_not_zero(self, field): """`value` is not within an acceptable threshold of `field.zero`.""" @@ -135,6 +142,7 @@ class TestIsZero: assert pytest.approx(field.zero, abs=utils.DEFAULT_THRESHOLD) != value assert not field.is_zero(value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) @pytest.mark.parametrize("value", utils.ONES) def test_is_not_zero(self, field, value): @@ -153,6 +161,7 @@ class TestIsOne: assert field.one == value assert field.is_one(value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) def test_is_almost_one(self, field): """`value` is within an acceptable threshold of `field.one`.""" @@ -161,6 +170,7 @@ class TestIsOne: assert pytest.approx(field.one, abs=utils.DEFAULT_THRESHOLD) == value assert field.is_one(value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.NON_10_FIELDS) def test_is_slightly_not_one(self, field): """`value` is not within an acceptable threshold of `field.one`.""" @@ -169,6 +179,7 @@ class TestIsOne: assert pytest.approx(field.one, abs=utils.DEFAULT_THRESHOLD) != value assert not field.is_one(value) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) @pytest.mark.parametrize("value", utils.ZEROS) def test_is_not_one(self, field, value): @@ -193,6 +204,7 @@ class TestDrawRandomFieldElement: assert field.validate(element) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.ALL_FIELDS) def test_draw_element_with_default_bounds_set_to_none(self, field): """Draw a random element from the `field`, ... @@ -206,6 +218,7 @@ class TestDrawRandomFieldElement: assert field.validate(element) + @pytest.mark.overlapping_test @pytest.mark.parametrize("field", utils.NON_10_FIELDS) def test_draw_element_with_custom_bounds(self, field): """Draw a random element from the `field` ... @@ -242,6 +255,7 @@ class TestDrawRandomFieldElement: assert lower <= element2 <= upper +@pytest.mark.sanity_test def test_numbers(): """We use `0`, `1`, `+42`, and `-42` in different data types.""" unique_one_and_zero = {int(n) for n in utils.ONES_N_ZEROS} diff --git a/tests/fields/test_complex.py b/tests/fields/test_complex.py index dc5b1f3..be9bc85 100644 --- a/tests/fields/test_complex.py +++ b/tests/fields/test_complex.py @@ -8,6 +8,10 @@ from lalib import fields from tests.fields import utils +# None of the test cases below contributes towards higher coverage +pytestmark = pytest.mark.overlapping_test + + C = fields.C diff --git a/tests/fields/test_galois.py b/tests/fields/test_galois.py index 5b73c73..be69dfc 100644 --- a/tests/fields/test_galois.py +++ b/tests/fields/test_galois.py @@ -8,6 +8,10 @@ from lalib import fields from tests.fields import utils +# None of the test cases below contributes towards higher coverage +pytestmark = pytest.mark.overlapping_test + + GF2 = fields.GF2 diff --git a/tests/fields/test_rational.py b/tests/fields/test_rational.py index 6d3c850..62b909b 100644 --- a/tests/fields/test_rational.py +++ b/tests/fields/test_rational.py @@ -7,6 +7,10 @@ import pytest from lalib import fields +# None of the test cases below contributes towards higher coverage +pytestmark = pytest.mark.overlapping_test + + Q = fields.Q diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 5eaf571..ce1ba38 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -10,6 +10,7 @@ import pytest import xdoctest +@pytest.mark.integration_test @pytest.mark.parametrize( "module", [ diff --git a/tests/test_top_level_imports.py b/tests/test_top_level_imports.py index 909e46d..fb5be32 100644 --- a/tests/test_top_level_imports.py +++ b/tests/test_top_level_imports.py @@ -6,6 +6,7 @@ from typing import Any import pytest +@pytest.mark.integration_test @pytest.mark.parametrize( "path_to_package", [ diff --git a/tests/test_version.py b/tests/test_version.py index acc3d6c..c6c6429 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -236,36 +236,7 @@ INVALID_NOT_SEMANTIC = ( INVALID_VERSIONS = INVALID_NOT_READABLE + INVALID_NOT_SEMANTIC -@pytest.mark.parametrize( - ["version1", "version2"], - zip( # loop over pairs of neighboring elements - VALID_AND_NORMALIZED_VERSIONS, - VALID_AND_NORMALIZED_VERSIONS[1:], - ), -) -def test_versions_are_strictly_ordered(version1, version2): - """`VALID_AND_NORMALIZED_VERSIONS` are ordered.""" - version1_parsed = pkg_version.Version(version1) - version2_parsed = pkg_version.Version(version2) - - assert version1_parsed < version2_parsed - - -@pytest.mark.parametrize( - ["version1", "version2"], - zip( # loop over pairs of neighboring elements - VALID_AND_NOT_NORMALIZED_VERSIONS, - VALID_AND_NOT_NORMALIZED_VERSIONS[1:], - ), -) -def test_versions_are_weakly_ordered(version1, version2): - """`VALID_AND_NOT_NORMALIZED_VERSIONS` are ordered.""" - version1_parsed = pkg_version.Version(version1) - version2_parsed = pkg_version.Version(version2) - - assert version1_parsed <= version2_parsed - - +@pytest.mark.overlapping_test class VersionClassification: """Classifying version identifiers. @@ -344,6 +315,7 @@ class VersionClassification: return is_so_by_parts +@pytest.mark.overlapping_test class TestVersionIdentifier(VersionClassification): """The versions must comply with PEP440 ... @@ -504,6 +476,7 @@ class TestVersionIdentifier(VersionClassification): assert parsed_version.public != unparsed_version +@pytest.mark.overlapping_test class TestVersionIdentifierWithPattern: """Test the versioning with a custom `regex` pattern.""" @@ -585,3 +558,36 @@ class TestUnavailablePackageMetadata: with self.hide_metadata_from_package("lalib") as lalib_pkg: assert lalib_pkg.__pkg_name__ == "unknown" assert lalib_pkg.__version__ == "unknown" + + +@pytest.mark.sanity_test +class TestSampleVersionData: + """Ensure the `VALID_*_VERSIONS` are in order.""" + + @pytest.mark.parametrize( + ["version1", "version2"], + zip( # loop over pairs of neighboring elements + VALID_AND_NORMALIZED_VERSIONS, + VALID_AND_NORMALIZED_VERSIONS[1:], + ), + ) + def test_versions_are_strictly_ordered(self, version1, version2): + """`VALID_AND_NORMALIZED_VERSIONS` are ordered.""" + version1_parsed = pkg_version.Version(version1) + version2_parsed = pkg_version.Version(version2) + + assert version1_parsed < version2_parsed + + @pytest.mark.parametrize( + ["version1", "version2"], + zip( # loop over pairs of neighboring elements + VALID_AND_NOT_NORMALIZED_VERSIONS, + VALID_AND_NOT_NORMALIZED_VERSIONS[1:], + ), + ) + def test_versions_are_weakly_ordered(self, version1, version2): + """`VALID_AND_NOT_NORMALIZED_VERSIONS` are ordered.""" + version1_parsed = pkg_version.Version(version1) + version2_parsed = pkg_version.Version(version2) + + assert version1_parsed <= version2_parsed