From da233e2e35445080c502ba47b5c34573e255b927 Mon Sep 17 00:00:00 2001 From: Alexander Hess Date: Tue, 4 Aug 2020 02:55:41 +0200 Subject: [PATCH] Set up pre-commit hooks - add pre-commit and pre-merge hooks: + run `poetry run nox -s pre-commit` on staged *.py files + run common pre-commit hooks for validations that could not be achieved with tools in the develop environment so easily + run `poetry run nox -s pre-merge` before merges and pushes - implement the "pre-commit" and "pre-merge" sessions in nox + include a little hack to deal with the positional arguments passed by the pre-commit framework - provide more documentation on the nox sessions --- .pre-commit-config.yaml | 38 ++++++++++++++++++ noxfile.py | 87 ++++++++++++++++++++++++++++++++++------- poetry.lock | 60 +++++++++++++++++++++++++++- pyproject.toml | 1 + 4 files changed, 171 insertions(+), 15 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cefbf7e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,38 @@ +default_stages: [commit] # only used if a hook does not specify stages +fail_fast: true +repos: +# Run the local formatting, linting, and testing tool chains. +- repo: local + hooks: + - id: local-pre-commit-checks + name: Run code formatters and linters + entry: poetry run nox -s pre-commit -- + language: system + stages: [commit] + types: [python] + - id: local-pre-merge-checks + name: Run the entire test suite + entry: poetry run nox -s pre-merge -- + language: system + stages: [merge-commit, push] + types: [python] +# Enable hooks provided by the pre-commit project to +# enforce rules that local tools could not that easily. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: check-added-large-files + args: [--maxkb=100] + - id: check-case-conflict + - id: check-builtin-literals + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + stages: [commit] # overwrite the default + - id: mixed-line-ending + args: [--fix=no] + - id: no-commit-to-branch + args: [--branch, main] + - id: trailing-whitespace + stages: [commit] # overwrite the default diff --git a/noxfile.py b/noxfile.py index 9b24133..a1a09d6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -39,7 +39,11 @@ nox.options.sessions = ( @nox.session(name='format', python=MAIN_PYTHON) def format_(session: Session) -> None: - """Format source files with autoflake, black, and isort.""" + """Format source files with autoflake, black, and isort. + + If no extra arguments are provided, all source files are formatted. + Otherwise, they are interpreted as paths the formatters work on recursively. + """ _begin(session) _install_packages(session, 'autoflake', 'black', 'isort') # Interpret extra arguments as locations of source files. @@ -65,7 +69,11 @@ def format_(session: Session) -> None: @nox.session(python=MAIN_PYTHON) def lint(session: Session) -> None: - """Lint source files with flake8, mypy, and pylint.""" + """Lint source files with flake8, mypy, and pylint. + + If no extra arguments are provided, all source files are linted. + Otherwise, they are interpreted as paths the linters work on recursively. + """ _begin(session) _install_packages( session, @@ -99,20 +107,33 @@ def lint(session: Session) -> None: @nox.session(python=[MAIN_PYTHON, NEXT_PYTHON]) def test(session: Session) -> None: - """Test the code base.""" + """Test the code base. + + Runs the unit and integration tests (written with pytest). + + If no extra arguments are provided, the entire test suite + is exexcuted and succeeds only with 100% coverage. + + If extra arguments are provided, they are + forwarded to pytest without any changes. + """ # Re-using an old environment is not so easy here as # `poetry install --no-dev` removes previously installed packages. # We keep things simple and forbid such usage. if session.virtualenv.reuse_existing: - raise RuntimeError('The "test" session must be run with the "-r" option') + raise RuntimeError( + 'The "test" and "pre-merge" sessions must be run without the "-r" option', + ) _begin(session) - # Install only the non-develop dependencies - # and the testing tool chain. + # Install only the non-develop dependencies and the testing tool chain. session.run('poetry', 'install', '--no-dev', external=True) _install_packages(session, 'pytest', 'pytest-cov') # Interpret extra arguments as options for pytest. - args = session.posargs or ( + # They are "dropped" by the hack in the pre_merge() function + # if this function is run within the "pre-merge" session. + posargs = () if session.env.get('_drop_posargs') else session.posargs + args = posargs or ( '--cov', '--no-cov-on-fail', '--cov-branch', @@ -125,23 +146,54 @@ def test(session: Session) -> None: @nox.session(name='pre-commit', python=MAIN_PYTHON, venv_backend='none') def pre_commit(session: Session) -> None: - """Source files must be well-formed before they enter git.""" - _begin(session) + """Source files must be well-formed before they enter git. + + Intended to be run as a pre-commit hook. + + This session is a wrapper that triggers the "format" and "lint" sessions. + + Passed in extra arguments are forwarded. So, if it is run as a pre-commit + hook, only the currently staged source files are formatted and linted. + """ + # "format" and "lint" are run in sessions on their own as + # session.notify() creates new Session objects. session.notify('format') session.notify('lint') -@nox.session(name='pre-merge', python=MAIN_PYTHON, venv_backend='none') +@nox.session(name='pre-merge', python=MAIN_PYTHON) def pre_merge(session: Session) -> None: - """The test suite must pass before merges are made.""" - _begin(session) - session.notify('test') + """The test suite must pass before merges are made. + + Intended to be run either as a pre-merge or pre-push hook. + + First, this session triggers the "format" and "lint" sessions via + the "pre-commit" session. + + Then, it runs the "test" session ignoring any extra arguments passed in + so that the entire test suite is executed. + """ + session.notify('pre-commit') + # Little hack to not work with the extra arguments provided + # by the pre-commit framework. Create a flag in the + # env(ironment) that must contain only `str`-like objects. + session.env['_drop_posargs'] = 'true' + # Cannot use session.notify() to trigger the "test" session + # as that would create a new Session object without the flag + # in the env(ironment). Instead, run the test() function within + # the "pre-merge" session. + test(session) def _begin(session: Session) -> None: """Show generic info about a session.""" if session.posargs: - print('extra arguments:', *session.posargs) # noqa:WPS421 + # Part of the hack in pre_merge() to "drop" the extra arguments. + # Indicate to stdout that the passed in extra arguments are ignored. + if session.env.get('_drop_posargs') is None: + print('Provided extra argument(s):', *session.posargs) # noqa:WPS421 + else: + print('The provided extra arguments are ignored') # noqa:WPS421 session.run('python', '--version') @@ -159,6 +211,13 @@ def _install_packages( packages respecting the pinnned versions specified in poetry's lock file. This makes nox sessions even more deterministic. + IMPORTANT: This function skips installation if the current nox session + is run with the "-r" flag to re-use an existing virtual environment. + That turns nox into a fast task runner provided that a virtual + environment actually existed and does not need to be changed (e.g., + new dependencies added in the meantime). Do not use the "-r" flag on CI + or as part of pre-commit hooks! + Args: session: the Session object *packages_or_pip_args: the packages to be installed or pip options diff --git a/poetry.lock b/poetry.lock index f4a669c..6e55d75 100644 --- a/poetry.lock +++ b/poetry.lock @@ -118,6 +118,14 @@ typed-ast = ">=1.4.0" [package.extras] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +[[package]] +category = "dev" +description = "Validate configuration and produce human readable error messages." +name = "cfgv" +optional = false +python-versions = ">=3.6.1" +version = "3.2.0" + [[package]] category = "dev" description = "Composable command line interface toolkit" @@ -447,6 +455,17 @@ version = "3.1.7" [package.dependencies] gitdb = ">=4.0.1,<5" +[[package]] +category = "dev" +description = "File identification library for Python" +name = "identify" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +version = "1.4.25" + +[package.extras] +license = ["editdistance"] + [[package]] category = "dev" description = "iniconfig: brain-dead simple config-ini parsing" @@ -517,6 +536,14 @@ optional = false python-versions = "*" version = "0.4.3" +[[package]] +category = "dev" +description = "Node.js virtual environment builder" +name = "nodeenv" +optional = false +python-versions = "*" +version = "1.4.0" + [[package]] category = "dev" description = "Flexible test automation." @@ -584,6 +611,22 @@ version = "0.13.1" [package.extras] dev = ["pre-commit", "tox"] +[[package]] +category = "dev" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +name = "pre-commit" +optional = false +python-versions = ">=3.6.1" +version = "2.6.0" + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +toml = "*" +virtualenv = ">=20.0.8" + [[package]] category = "dev" description = "library with cross-python path, ini-parsing, io, code, log facilities" @@ -843,7 +886,7 @@ python-versions = "*" version = "1.12.1" [metadata] -content-hash = "1a6ddbc4c05cb41329cbac4792210897f11dda8e8ba3aa7e814a4a0d0d9c4fa8" +content-hash = "7a843263817a908ca01d198402d3e9310c33307ac85085f028c8fbdd7587f48f" lock-version = "1.0" python-versions = "^3.8" @@ -887,6 +930,10 @@ black = [ {file = "black-19.10b0-py36-none-any.whl", hash = "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b"}, {file = "black-19.10b0.tar.gz", hash = "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"}, ] +cfgv = [ + {file = "cfgv-3.2.0-py2.py3-none-any.whl", hash = "sha256:32e43d604bbe7896fe7c248a9c2276447dbef840feb28fe20494f62af110211d"}, + {file = "cfgv-3.2.0.tar.gz", hash = "sha256:cf22deb93d4bcf92f345a5c3cd39d3d41d6340adc60c78bbbd6588c384fda6a1"}, +] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, @@ -1032,6 +1079,10 @@ gitpython = [ {file = "GitPython-3.1.7-py3-none-any.whl", hash = "sha256:fa3b92da728a457dd75d62bb5f3eb2816d99a7fe6c67398e260637a40e3fafb5"}, {file = "GitPython-3.1.7.tar.gz", hash = "sha256:2db287d71a284e22e5c2846042d0602465c7434d910406990d5b74df4afb0858"}, ] +identify = [ + {file = "identify-1.4.25-py2.py3-none-any.whl", hash = "sha256:ccd88716b890ecbe10920659450a635d2d25de499b9a638525a48b48261d989b"}, + {file = "identify-1.4.25.tar.gz", hash = "sha256:110ed090fec6bce1aabe3c72d9258a9de82207adeaa5a05cd75c635880312f9a"}, +] iniconfig = [ {file = "iniconfig-1.0.1-py3-none-any.whl", hash = "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437"}, {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"}, @@ -1091,6 +1142,9 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] +nodeenv = [ + {file = "nodeenv-1.4.0-py2.py3-none-any.whl", hash = "sha256:4b0b77afa3ba9b54f4b6396e60b0c83f59eaeb2d63dc3cc7a70f7f4af96c82bc"}, +] nox = [ {file = "nox-2020.5.24-py3-none-any.whl", hash = "sha256:c4509621fead99473a1401870e680b0aadadce5c88440f0532863595176d64c1"}, {file = "nox-2020.5.24.tar.gz", hash = "sha256:61a55705736a1a73efbd18d5b262a43d55a1176546e0eb28b29064cfcffe26c0"}, @@ -1115,6 +1169,10 @@ pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] +pre-commit = [ + {file = "pre_commit-2.6.0-py2.py3-none-any.whl", hash = "sha256:e8b1315c585052e729ab7e99dcca5698266bedce9067d21dc909c23e3ceed626"}, + {file = "pre_commit-2.6.0.tar.gz", hash = "sha256:1657663fdd63a321a4a739915d7d03baedd555b25054449090f97bb0cb30a915"}, +] py = [ {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, diff --git a/pyproject.toml b/pyproject.toml index c4934e0..ebc9618 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ python = "^3.8" [tool.poetry.dev-dependencies] # Task Runners nox = "^2020.5.24" +pre-commit = "^2.6.0" # Code Formatters autoflake = "^1.3.1"