alexander/lalib
1
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Add smoke tests

Browse source 
- extend `pytest` with an option to run only the minimum number
  of (unit) test cases to just keep the coverage at 100%
- rationale:
  + many of the unit test cases partly overlap with
    respect to the lines of source code executed
  + also, integration tests, by definition, do not
    contribute to a higher test coverage
- implementation: mark "redundant" test cases as one of:
  + `pytest.mark.integration_test`
    => code usage from the perspective of the end user
  + `pytest.mark.overlapping_test`
    => tests not contributing to the 100% coverage
  + `pytest.mark.sanity_test`
    => tests providing confidence in the test data
- add `tests.conftest` module
  => programatically convert the above markers into
     `@pytest.mark.no_cover` and collect the non-"redundant" tests
- add nox session "test-fast" to run only the minimum
  number of (unit) test while holding coverage at 100%
- refactor some test modules
  + wrap some test cases in a class
  + move sanity tests to the end of the files
This commit is contained in:
Alexander Hess 2024-10-15 01:49:32 +02:00
parent de740ebb5f
commit 7e3e67c300
Signed by: alexander
GPG key ID: 344EA5AB10D868E0
11 changed files with 182 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
noxfile.py
View file

@ -254,6 +254,14 @@ def test(session: nox.Session) -> None:
args = posargs or (
"--cov",
"--no-cov-on-fail",
*( # If this function is run via the "test-fast" session,
( # the following arguments are added:
"--cov-fail-under=100",
"--smoke-tests-only",
)
if session.env.get("_smoke_tests_only")
else ()
),
TEST_RANDOM_SEED,
TESTS_LOCATION,
)
@ -339,6 +347,18 @@ def test_docstrings(session: nox.Session) -> None:
session.run("xdoctest", "src/lalib")
@nox.session(name="test-fast", python=MAIN_PYTHON)
def test_fast(session: nox.Session) -> None:
"""Test code with `pytest`, selected (smoke) tests only.
The (unit) test cases are selected such that their number
is minimal but the achieved coverage remains at 100%.
"""
# See implementation notes in `pre_commit_test_hook()` below
session.env["_smoke_tests_only"] = "true"
test(session)
@nox.session(name="clean-cwd", python=MAIN_PYTHON, venv_backend="none")
def clean_cwd(session: nox.Session) -> None:
"""Remove (almost) all glob patterns listed in git's ignore file.

59
tests/conftest.py Normal file
View file

@ -0,0 +1,59 @@
"""Configurations and utilities for all tests."""
import pytest
def pytest_addoption(parser):
"""Define custom CLI options for `pytest`."""
parser.addoption(
"--smoke-tests-only",
action="store_true",
default=False,
help="Run the minimum number of (unit) tests to achieve full coverage",
)
def pytest_configure(config):
"""Define custom markers explicitly."""
config.addinivalue_line(
"markers",
"integration_test: non-unit test case; skipped during coverage reporting",
)
config.addinivalue_line(
"markers",
"overlapping_test: test case not contributing towards higher coverage",
)
config.addinivalue_line(
"markers",
"sanity_test: test case providing confidence in the test data",
)
def pytest_collection_modifyitems(config, items):
"""Pre-process the test cases programatically.
- Add `no_cover` marker to test cases with any of these markers:
+ "integration_test"
+ "overlapping_test"
+ "sanity_test"
- Select test cases with none of the above markers as smoke tests,
i.e., the minimum number of test cases to achieve 100% coverage
"""
smoke_tests = []
for item in items:
if (
"integration_test" in item.keywords
or "overlapping_test" in item.keywords
or "sanity_test" in item.keywords
):
item.add_marker(pytest.mark.no_cover)
elif config.getoption("--smoke-tests-only"):
smoke_tests.append(item)
if config.getoption("--smoke-tests-only"):
if not smoke_tests:
pytest.exit("No smoke tests found")
items[:] = smoke_tests

40
tests/elements/test_galois.py
View file

@ -78,11 +78,7 @@ zero_like_values = (
)
def test_thresholds():
"""Sanity check for the thresholds used in the tests below."""
assert utils.WITHIN_THRESHOLD < utils.DEFAULT_THRESHOLD < utils.NOT_WITHIN_THRESHOLD
@pytest.mark.overlapping_test
class TestGF2SubClasses:
"""Test the sub-classes behind `one` and `zero`."""
@ -122,6 +118,7 @@ class TestGF2Casting:
result = cls(value, strict=False)
assert result is one
@pytest.mark.overlapping_test
@pytest.mark.parametrize("value", non_strict_one_like_values)
def test_cannot_cast_ones_strictly(self, cls, value):
"""`gf2(value, strict=False)` returns `1`."""
@ -156,6 +153,7 @@ class TestGF2Casting:
with pytest.raises(ValueError, match="`1`-like or `0`-like"):
cls(value, strict=strict)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("value", ["abc", (1,), [1]])
@pytest.mark.parametrize("strict", [True, False])
def test_cannot_cast_from_wrong_type(self, cls, value, strict):
@ -163,12 +161,14 @@ class TestGF2Casting:
with pytest.raises(TypeError):
cls(value, strict=strict)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("strict", [True, False])
def test_cannot_cast_from_nan_value(self, cls, strict):
"""Cannot create `one` or `zero` from undefined value."""
with pytest.raises(ValueError, match="`1`-like or `0`-like"):
cls(float("NaN"), strict=strict)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("scaler", [1, 10, 100, 1000])
def test_get_one_if_within_threshold(self, cls, scaler):
"""`gf2()` returns `one` if `value` is larger than `threshold`."""
@ -180,6 +180,7 @@ class TestGF2Casting:
result = cls(value, strict=False, threshold=threshold)
assert result is one
@pytest.mark.overlapping_test
@pytest.mark.parametrize("scaler", [1, 10, 100, 1000])
@pytest.mark.parametrize("strict", [True, False])
def test_get_zero_if_within_threshold(self, cls, scaler, strict):
@ -192,6 +193,7 @@ class TestGF2Casting:
assert result is zero
@pytest.mark.overlapping_test
@pytest.mark.parametrize("strict", [True, False])
class TestGF2ConstructorWithoutCastedValue:
"""Test the `gf2` class's constructor.
@ -220,6 +222,7 @@ class TestGenericBehavior:
with pytest.raises(RuntimeError, match="internal error"):
gf2()
@pytest.mark.overlapping_test
@pytest.mark.parametrize("cls", [gf2, GF2One, GF2Zero])
def test_create_singletons(self, cls):
"""Singleton pattern: The classes always return the same instance."""
@ -227,6 +230,7 @@ class TestGenericBehavior:
second = cls()
assert first is second
@pytest.mark.overlapping_test
@pytest.mark.parametrize("obj", [one, zero])
def test_sub_classes_return_objs(self, obj):
"""`type(one)` and `type(zero)` return ...
@ -239,6 +243,7 @@ class TestGenericBehavior:
new_obj = sub_cls()
assert new_obj is obj
@pytest.mark.overlapping_test
@pytest.mark.parametrize("obj", [one, zero])
@pytest.mark.parametrize(
"type_",
@ -253,6 +258,7 @@ class TestGenericBehavior:
"""`one` and `zero` are officially `Numbers`s."""
assert isinstance(obj, type_)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("cls", [gf2, GF2One, GF2Zero])
@pytest.mark.parametrize(
"method",
@ -320,14 +326,17 @@ class TestGenericBehavior:
class TestNumericBehavior:
"""Test how `one` and `zero` behave like numbers."""
@pytest.mark.overlapping_test
def test_make_complex(self):
"""`one` and `zero` behave like `1 + 0j` and `0 + 0j`."""
assert (complex(one), complex(zero)) == (1 + 0j, 0 + 0j)
@pytest.mark.overlapping_test
def test_make_float(self):
"""`one` and `zero` behave like `1.0` and `0.0`."""
assert (float(one), float(zero)) == (1.0, 0.0)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("func", [int, hash])
def test_make_int(self, func):
"""`one` and `zero` behave like `1` and `0`.
@ -340,6 +349,7 @@ class TestNumericBehavior:
"""`one` and `zero` behave like `True` and `False`."""
assert (bool(one), bool(zero)) == (True, False)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("obj", [one, zero])
def test_get_abs_value(self, obj):
"""`abs(one)` and `abs(zero)` are `one` and `zero`."""
@ -366,10 +376,12 @@ class TestNumericBehavior:
"""`one.conjugate()` and `zero.conjugate()` are `1 + 0j` and `0 + 0j`."""
assert (one.conjugate(), zero.conjugate()) == (1 + 0j, 0 + 0j)
@pytest.mark.overlapping_test
def test_one_as_fraction(self):
"""`one.numerator / one.denominator` equals `1`."""
assert (one.numerator, one.denominator) == (1, 1)
@pytest.mark.overlapping_test
def test_zero_as_fraction(self):
"""`one.numerator / one.denominator` equals `0`."""
assert (zero.numerator, zero.denominator) == (0, 1)
@ -378,11 +390,13 @@ class TestNumericBehavior:
class TestComparison:
"""Test `one` and `zero` interact with relational operators."""
@pytest.mark.overlapping_test
@pytest.mark.parametrize("obj", [one, zero])
def test_equal_to_itself(self, obj):
"""`one` and `zero` are equal to themselves."""
assert obj == obj # noqa: PLR0124
@pytest.mark.overlapping_test
@pytest.mark.parametrize(
["first", "second"],
[
@ -401,6 +415,7 @@ class TestComparison:
assert first == second
assert second == first
@pytest.mark.overlapping_test
@pytest.mark.parametrize(
["first", "second"],
[
@ -418,6 +433,7 @@ class TestComparison:
assert first != second
assert second != first
@pytest.mark.overlapping_test
@pytest.mark.parametrize(
["first", "second"],
[
@ -441,27 +457,32 @@ class TestComparison:
"""`one > zero` and `one >= zero`."""
assert operator(one, zero)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("operator", [operator.lt, operator.le])
def test_one_not_smaller_than_or_equal_to_zero(self, operator):
"""`not one < zero` and `not one <= zero`."""
assert not operator(one, zero)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("operator", [operator.lt, operator.le])
def test_zero_smaller_than_or_equal_to_one(self, operator):
"""`zero < one` and `zero <= one`."""
assert operator(zero, one)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("operator", [operator.gt, operator.ge])
def test_zero_not_greater_than_or_equalt_to_one(self, operator):
"""`not zero > one` and `not zero >= one`."""
assert not operator(zero, one)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("obj", [one, zero])
def test_obj_not_strictly_greater_than_itself(self, obj):
"""`obj >= obj` but not `obj > obj`."""
assert obj >= obj # noqa: PLR0124
assert not obj > obj # noqa: PLR0124
@pytest.mark.overlapping_test
@pytest.mark.parametrize("obj", [one, zero])
def test_obj_not_strictly_smaller_than_itself(self, obj):
"""`obj <= obj` but not `obj < obj`."""
@ -579,6 +600,7 @@ class TestArithmetic:
result4 = gf2(int(abs(second)) * int(abs(first)))
assert result4 is expected
@pytest.mark.overlapping_test
@pytest.mark.parametrize(
"objs",
[
@ -613,6 +635,7 @@ class TestArithmetic:
result2 = gf2(operator(int(abs(first)), int(abs(second))))
assert result2 is expected
@pytest.mark.overlapping_test
@pytest.mark.parametrize(
"objs",
[
@ -762,6 +785,7 @@ class TestArithmetic:
operator(42, obj)
@pytest.mark.overlapping_test
@pytest.mark.skipif(
not sys.version_info < (3, 11),
reason='"typing-extensions" are installed to support Python 3.9 & 3.10',
@ -772,3 +796,9 @@ def test_can_import_typing_extensions():
importlib.reload(package)
assert package.Self is not None
@pytest.mark.sanity_test
def test_thresholds():
"""Sanity check for the thresholds used in the tests below."""
assert utils.WITHIN_THRESHOLD < utils.DEFAULT_THRESHOLD < utils.NOT_WITHIN_THRESHOLD

4
tests/fields/test_axioms.py
View file

@ -11,6 +11,10 @@ import pytest
from tests.fields import utils
# None of the test cases below contributes towards higher coverage
pytestmark = pytest.mark.integration_test
@pytest.mark.repeat(utils.N_RANDOM_DRAWS)
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
class TestAllFieldsManyTimes:

14
tests/fields/test_base.py
View file

@ -39,6 +39,7 @@ class TestCastAndValidateFieldElements:
an element of the `field`, and, if so, `.cast()` it as such.
"""
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.NON_10_FIELDS)
@pytest.mark.parametrize("value", utils.NUMBERS)
def test_number_is_field_element(self, field, value):
@ -49,12 +50,14 @@ class TestCastAndValidateFieldElements:
"""
utils.is_field_element(field, value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
@pytest.mark.parametrize("value", utils.ONES_N_ZEROS)
def test_one_and_zero_number_is_field_element(self, field, value):
"""`1`-like and `0`-like numbers are always `field` elements."""
utils.is_field_element(field, value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
@pytest.mark.parametrize("value", ["abc", (1, 2, 3)])
def test_non_numeric_value_is_not_field_element(self, field, value):
@ -93,6 +96,7 @@ class TestDTypes:
"""`field.dtype` must be a `type`."""
assert isinstance(field.dtype, type)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
def test_element_is_instance_of_field_dtype(self, field):
"""Elements are an instance of `field.dtype`."""
@ -100,6 +104,7 @@ class TestDTypes:
assert isinstance(element, field.dtype)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
def test_element_dtype_is_subclass_of_field_dtype(self, field):
"""Elements may have a more specific `.dtype` than their `field.dtype`."""
@ -119,6 +124,7 @@ class TestIsZero:
assert field.zero == value
assert field.is_zero(value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
def test_is_almost_zero(self, field):
"""`value` is within an acceptable threshold of `field.zero`."""
@ -127,6 +133,7 @@ class TestIsZero:
assert pytest.approx(field.zero, abs=utils.DEFAULT_THRESHOLD) == value
assert field.is_zero(value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.NON_10_FIELDS)
def test_is_slightly_not_zero(self, field):
"""`value` is not within an acceptable threshold of `field.zero`."""
@ -135,6 +142,7 @@ class TestIsZero:
assert pytest.approx(field.zero, abs=utils.DEFAULT_THRESHOLD) != value
assert not field.is_zero(value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
@pytest.mark.parametrize("value", utils.ONES)
def test_is_not_zero(self, field, value):
@ -153,6 +161,7 @@ class TestIsOne:
assert field.one == value
assert field.is_one(value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
def test_is_almost_one(self, field):
"""`value` is within an acceptable threshold of `field.one`."""
@ -161,6 +170,7 @@ class TestIsOne:
assert pytest.approx(field.one, abs=utils.DEFAULT_THRESHOLD) == value
assert field.is_one(value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.NON_10_FIELDS)
def test_is_slightly_not_one(self, field):
"""`value` is not within an acceptable threshold of `field.one`."""
@ -169,6 +179,7 @@ class TestIsOne:
assert pytest.approx(field.one, abs=utils.DEFAULT_THRESHOLD) != value
assert not field.is_one(value)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
@pytest.mark.parametrize("value", utils.ZEROS)
def test_is_not_one(self, field, value):
@ -193,6 +204,7 @@ class TestDrawRandomFieldElement:
assert field.validate(element)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.ALL_FIELDS)
def test_draw_element_with_default_bounds_set_to_none(self, field):
"""Draw a random element from the `field`, ...
@ -206,6 +218,7 @@ class TestDrawRandomFieldElement:
assert field.validate(element)
@pytest.mark.overlapping_test
@pytest.mark.parametrize("field", utils.NON_10_FIELDS)
def test_draw_element_with_custom_bounds(self, field):
"""Draw a random element from the `field` ...
@ -242,6 +255,7 @@ class TestDrawRandomFieldElement:
assert lower <= element2 <= upper
@pytest.mark.sanity_test
def test_numbers():
"""We use `0`, `1`, `+42`, and `-42` in different data types."""
unique_one_and_zero = {int(n) for n in utils.ONES_N_ZEROS}

4
tests/fields/test_complex.py
View file

@ -8,6 +8,10 @@ from lalib import fields
from tests.fields import utils
# None of the test cases below contributes towards higher coverage
pytestmark = pytest.mark.overlapping_test
C = fields.C

4
tests/fields/test_galois.py
View file

@ -8,6 +8,10 @@ from lalib import fields
from tests.fields import utils
# None of the test cases below contributes towards higher coverage
pytestmark = pytest.mark.overlapping_test
GF2 = fields.GF2

4
tests/fields/test_rational.py
View file

@ -7,6 +7,10 @@ import pytest
from lalib import fields
# None of the test cases below contributes towards higher coverage
pytestmark = pytest.mark.overlapping_test
Q = fields.Q

1
tests/test_docstrings.py
View file

@ -10,6 +10,7 @@ import pytest
import xdoctest
@pytest.mark.integration_test
@pytest.mark.parametrize(
"module",
[

1
tests/test_top_level_imports.py
View file

@ -6,6 +6,7 @@ from typing import Any
import pytest
@pytest.mark.integration_test
@pytest.mark.parametrize(
"path_to_package",
[

66
tests/test_version.py
View file

@ -236,36 +236,7 @@ INVALID_NOT_SEMANTIC = (
INVALID_VERSIONS = INVALID_NOT_READABLE + INVALID_NOT_SEMANTIC
@pytest.mark.parametrize(
["version1", "version2"],
zip( # loop over pairs of neighboring elements
VALID_AND_NORMALIZED_VERSIONS,
VALID_AND_NORMALIZED_VERSIONS[1:],
),
)
def test_versions_are_strictly_ordered(version1, version2):
"""`VALID_AND_NORMALIZED_VERSIONS` are ordered."""
version1_parsed = pkg_version.Version(version1)
version2_parsed = pkg_version.Version(version2)
assert version1_parsed < version2_parsed
@pytest.mark.parametrize(
["version1", "version2"],
zip( # loop over pairs of neighboring elements
VALID_AND_NOT_NORMALIZED_VERSIONS,
VALID_AND_NOT_NORMALIZED_VERSIONS[1:],
),
)
def test_versions_are_weakly_ordered(version1, version2):
"""`VALID_AND_NOT_NORMALIZED_VERSIONS` are ordered."""
version1_parsed = pkg_version.Version(version1)
version2_parsed = pkg_version.Version(version2)
assert version1_parsed <= version2_parsed
@pytest.mark.overlapping_test
class VersionClassification:
"""Classifying version identifiers.
@ -344,6 +315,7 @@ class VersionClassification:
return is_so_by_parts
@pytest.mark.overlapping_test
class TestVersionIdentifier(VersionClassification):
"""The versions must comply with PEP440 ...
@ -504,6 +476,7 @@ class TestVersionIdentifier(VersionClassification):
assert parsed_version.public != unparsed_version
@pytest.mark.overlapping_test
class TestVersionIdentifierWithPattern:
"""Test the versioning with a custom `regex` pattern."""
@ -585,3 +558,36 @@ class TestUnavailablePackageMetadata:
with self.hide_metadata_from_package("lalib") as lalib_pkg:
assert lalib_pkg.__pkg_name__ == "unknown"
assert lalib_pkg.__version__ == "unknown"
@pytest.mark.sanity_test
class TestSampleVersionData:
"""Ensure the `VALID_*_VERSIONS` are in order."""
@pytest.mark.parametrize(
["version1", "version2"],
zip( # loop over pairs of neighboring elements
VALID_AND_NORMALIZED_VERSIONS,
VALID_AND_NORMALIZED_VERSIONS[1:],
),
)
def test_versions_are_strictly_ordered(self, version1, version2):
"""`VALID_AND_NORMALIZED_VERSIONS` are ordered."""
version1_parsed = pkg_version.Version(version1)
version2_parsed = pkg_version.Version(version2)
assert version1_parsed < version2_parsed
@pytest.mark.parametrize(
["version1", "version2"],
zip( # loop over pairs of neighboring elements
VALID_AND_NOT_NORMALIZED_VERSIONS,
VALID_AND_NOT_NORMALIZED_VERSIONS[1:],
),
)
def test_versions_are_weakly_ordered(self, version1, version2):
"""`VALID_AND_NOT_NORMALIZED_VERSIONS` are ordered."""
version1_parsed = pkg_version.Version(version1)
version2_parsed = pkg_version.Version(version2)
assert version1_parsed <= version2_parsed