Add rpy2 to the dependencies

- add a Jupyter notebook that allows to install all project-external
  dependencies regarding R and R packages
- adjust the GitHub Action workflow to also install R and the R packages
  used within the project
- add a `init_r` module that initializes all R packages globally
  once the `urban_meal_delivery` package is imported

.github/workflows/tests.yml

@@ -1,7 +1,8 @@
 name: CI
 on: push
 jobs:
-  tests:
+  fast-tests:
+    name: fast (without R)
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -10,5 +11,22 @@ jobs:
         python-version: 3.8
         architecture: x64
     - run: pip install nox==2020.5.24
-    - run: pip install poetry==1.0.10
+    - run: pip install poetry==1.1.4
-    - run: nox
+    - run: nox -s format lint ci-tests-fast safety docs
+  slow-tests:
+    name: slow (with R)
+    runs-on: ubuntu-latest
+    env:
+      R_LIBS: .r_libs
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+        architecture: x64
+    - run: mkdir .r_libs
+    - run: sudo apt-get install r-base r-base-dev libcurl4-openssl-dev libxml2-dev patchelf
+    - run: R -e "install.packages('forecast')"
+    - run: pip install nox==2020.5.24
+    - run: pip install poetry==1.1.4
+    - run: nox -s ci-tests-slow

noxfile.py

@@ -25,26 +25,6 @@ as unified tasks to assure the quality of the source code:
   + accepts extra arguments, e.g., `poetry run nox -s test -- --no-cov`,
     that are passed on to `pytest` and `xdoctest` with no changes
     => may be paths or options
-
-
-GitHub Actions implements the following CI workflow:
-
-- "format", "lint", and "test" as above
-
-- "safety": check if dependencies contain known security vulnerabilites
-
-- "docs": build the documentation with sphinx
-
-
-The pre-commit framework invokes the following tasks:
-
-- before any commit:
-
-  + "format" and "lint" as above
-  + "fix-branch-references": replace branch references with the current one
-
-- before merges: run the entire "test-suite" independent of the file changes
-
 """
 
 import contextlib
@@ -92,7 +72,7 @@ nox.options.envdir = '.cache/nox'
 # Avoid accidental successes if the environment is not set up properly.
 nox.options.error_on_external_run = True
 
-# Run only CI related checks by default.
+# Run only local checks by default.
 nox.options.sessions = (
     'format',
     'lint',
@@ -220,24 +200,50 @@ def test(session):
         'xdoctest[optional]',
     )
 
+    session.run('pytest', '--version')
+
+    # When the CI server runs the slow tests, we only execute the R related
+    # test cases that require the slow installation of R and some packages.
+    if session.env.get('_slow_ci_tests'):
+        session.run(
+            'pytest', '--randomly-seed=4287', '-m', 'r', PYTEST_LOCATION,
+        )
+
+        # In the "ci-tests-slow" session, we do not run any test tool
+        # other than pytest. So, xdoctest, for example, is only run
+        # locally or in the "ci-tests-fast" session.
+        return
+
+    # When the CI server executes pytest, no database is available.
+    # Therefore, the CI server does not measure coverage.
+    elif session.env.get('_fast_ci_tests'):
+        pytest_args = (
+            '--randomly-seed=4287',
+            '-m',
+            'not (db or r)',
+            PYTEST_LOCATION,
+        )
+
+    # When pytest is executed in the local develop environment,
+    # both R and a database are available.
+    # Therefore, we require 100% coverage.
+    else:
+        pytest_args = (
+            '--cov',
+            '--no-cov-on-fail',
+            '--cov-branch',
+            '--cov-fail-under=100',
+            '--cov-report=term-missing:skip-covered',
+            '--randomly-seed=4287',
+            PYTEST_LOCATION,
+        )
+
     # Interpret extra arguments as options for pytest.
-    # They are "dropped" by the hack in the pre_merge() function
+    # They are "dropped" by the hack in the test_suite() function
-    # if this function is run within the "pre-merge" session.
+    # if this function is run within the "test-suite" session.
     posargs = () if session.env.get('_drop_posargs') else session.posargs
 
-    args = posargs or (
+    session.run('pytest', *(posargs or pytest_args))
-        '--cov',
-        '--no-cov-on-fail',
-        '--cov-branch',
-        '--cov-fail-under=100',
-        '--cov-report=term-missing:skip-covered',
-        '--randomly-seed=4287',
-        '-m',
-        'not (db or e2e)',
-        PYTEST_LOCATION,
-    )
-    session.run('pytest', '--version')
-    session.run('pytest', *args)
 
     # For xdoctest, the default arguments are different from pytest.
     args = posargs or [PACKAGE_IMPORT_NAME]
@@ -301,6 +307,60 @@ def docs(session):
     print(f'Docs are available at {os.getcwd()}/{DOCS_BUILD}index.html')  # noqa:WPS421
 
 
+@nox.session(name='ci-tests-fast', python=PYTHON)
+def fast_ci_tests(session):
+    """Fast tests run by the GitHub Actions CI server.
+
+    These regards all test cases NOT involving R via `rpy2`.
+
+    Also, coverage is not measured as full coverage can only be
+    achieved by running the tests in the local develop environment
+    that has access to a database.
+    """
+    # Re-using an old environment is not so easy here as the "test" session
+    # runs `poetry install --no-dev`, which removes previously installed packages.
+    if session.virtualenv.reuse_existing:
+        raise RuntimeError(
+            'The "ci-tests-fast" session must be run without the "-r" option',
+        )
+
+    # Little hack to pass arguments to the "test" session.
+    session.env['_fast_ci_tests'] = 'true'
+
+    # Cannot use session.notify() to trigger the "test" session
+    # as that would create a new Session object without the flag
+    # in the env(ironment).
+    test(session)
+
+
+@nox.session(name='ci-tests-slow', python=PYTHON)
+def slow_ci_tests(session):
+    """Slow tests run by the GitHub Actions CI server.
+
+    These regards all test cases involving R via `rpy2`.
+    They are slow as the CI server needs to install R and some packages
+    first, which takes a couple of minutes.
+
+    Also, coverage is not measured as full coverage can only be
+    achieved by running the tests in the local develop environment
+    that has access to a database.
+    """
+    # Re-using an old environment is not so easy here as the "test" session
+    # runs `poetry install --no-dev`, which removes previously installed packages.
+    if session.virtualenv.reuse_existing:
+        raise RuntimeError(
+            'The "ci-tests-slow" session must be run without the "-r" option',
+        )
+
+    # Little hack to pass arguments to the "test" session.
+    session.env['_slow_ci_tests'] = 'true'
+
+    # Cannot use session.notify() to trigger the "test" session
+    # as that would create a new Session object without the flag
+    # in the env(ironment).
+    test(session)
+
+
 @nox.session(name='test-suite', python=PYTHON)
 def test_suite(session):
     """Run the entire test suite.
@@ -324,8 +384,7 @@ def test_suite(session):
 
     # Cannot use session.notify() to trigger the "test" session
     # as that would create a new Session object without the flag
-    # in the env(ironment). Instead, run the test() function within
+    # in the env(ironment).
-    # the "pre-merge" session.
     test(session)
 
 

poetry.lock

@@ -95,7 +95,7 @@ python-versions = ">=3.5"
 name = "atomicwrites"
 version = "1.4.0"
 description = "Atomic file writes."
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
@@ -204,7 +204,7 @@ name = "cffi"
 version = "1.14.4"
 description = "Foreign Function Interface for Python calling C code."
 category = "main"
-optional = true
+optional = false
 python-versions = "*"
 
 [package.dependencies]
@@ -660,7 +660,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 name = "iniconfig"
 version = "1.1.1"
 description = "iniconfig: brain-dead simple config-ini parsing"
-category = "dev"
+category = "main"
 optional = false
 python-versions = "*"
 
@@ -1080,7 +1080,7 @@ name = "numpy"
 version = "1.19.4"
 description = "NumPy is the fundamental package for array computing with Python."
 category = "main"
-optional = true
+optional = false
 python-versions = ">=3.6"
 
 [[package]]
@@ -1179,7 +1179,7 @@ python-versions = "*"
 name = "pluggy"
 version = "0.13.1"
 description = "plugin and hook calling mechanisms for python"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
@@ -1261,7 +1261,7 @@ name = "pycparser"
 version = "2.20"
 description = "C parser in Python"
 category = "main"
-optional = true
+optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
 [[package]]
@@ -1311,7 +1311,7 @@ python-versions = ">=3.5"
 name = "pytest"
 version = "6.2.1"
 description = "pytest: simple powerful testing with Python"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.6"
 
@@ -1465,6 +1465,21 @@ python-versions = "*"
 [package.dependencies]
 docutils = ">=0.11,<1.0"
 
+[[package]]
+name = "rpy2"
+version = "3.4.1"
+description = "Python interface to the R language (embedded R)"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+cffi = ">=1.10.0"
+jinja2 = "*"
+pytest = "*"
+pytz = "*"
+tzlocal = "*"
+
 [[package]]
 name = "send2trash"
 version = "1.5.0"
@@ -1707,7 +1722,7 @@ python-versions = "*"
 name = "toml"
 version = "0.10.2"
 description = "Python Library for Tom's Obvious, Minimal Language"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 
@@ -1749,6 +1764,17 @@ category = "dev"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "tzlocal"
+version = "2.1"
+description = "tzinfo object for the local timezone"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pytz = "*"
+
 [[package]]
 name = "urllib3"
 version = "1.26.2"
@@ -1857,7 +1883,7 @@ research = ["jupyterlab", "nb_black", "numpy", "pytz"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "5c0a4b37e73e0ed607cc2c46a9178f7b8e2a8364856a408a80f955a3c8b861a1"
+content-hash = "9be7d168525c85958389c8edb4686567cbb4de0e8780168b91e387e1b0581ec3"
 
 [metadata.files]
 alabaster = [
@@ -2643,6 +2669,12 @@ requests = [
 restructuredtext-lint = [
     {file = "restructuredtext_lint-1.3.2.tar.gz", hash = "sha256:d3b10a1fe2ecac537e51ae6d151b223b78de9fafdd50e5eb6b08c243df173c80"},
 ]
+rpy2 = [
+    {file = "rpy2-3.4.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3f2d56bc80c2af0fe8118c53da7fd29f1809bc159a88cb10f9e2869321a21deb"},
+    {file = "rpy2-3.4.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:344ac89c966b2ec91bbf9e623b7ff9c121820b5e53da2ffc75fa10f158023cd7"},
+    {file = "rpy2-3.4.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ebbd7fceef359279f56b481d7ea2dd60db91928abb3726010a88fbb3362213af"},
+    {file = "rpy2-3.4.1.tar.gz", hash = "sha256:644360b569656700dfe13f59878ec1cf8c116c128d4f2f0bf96144031f95d2e2"},
+]
 send2trash = [
     {file = "Send2Trash-1.5.0-py3-none-any.whl", hash = "sha256:f1691922577b6fa12821234aeb57599d887c4900b9ca537948d2dac34aea888b"},
     {file = "Send2Trash-1.5.0.tar.gz", hash = "sha256:60001cc07d707fe247c94f74ca6ac0d3255aabcb930529690897ca2a39db28b2"},
@@ -2863,6 +2895,10 @@ typing-extensions = [
     {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"},
     {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"},
 ]
+tzlocal = [
+    {file = "tzlocal-2.1-py2.py3-none-any.whl", hash = "sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4"},
+    {file = "tzlocal-2.1.tar.gz", hash = "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44"},
+]
 urllib3 = [
     {file = "urllib3-1.26.2-py2.py3-none-any.whl", hash = "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"},
     {file = "urllib3-1.26.2.tar.gz", hash = "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08"},

pyproject.toml

@@ -42,6 +42,7 @@ jupyterlab = { version="^2.2.2", optional=true }
 nb_black = { version="^1.0.7", optional=true }
 numpy = { version="^1.19.1", optional=true }
 pytz = { version="^2020.1", optional=true }
+rpy2 = "^3.4.1"
 
 [tool.poetry.extras]
 research = [

setup.cfg

@@ -255,6 +255,8 @@ ignore_missing_imports = true
 ignore_missing_imports = true
 [mypy-pytest]
 ignore_missing_imports = true
+[mypy-rpy2.*]
+ignore_missing_imports = true
 [mypy-sqlalchemy.*]
 ignore_missing_imports = true
 [mypy-utm.*]
@@ -269,5 +271,6 @@ console_output_style = count
 env =
     TESTING=true
 markers =
-    db: tests touching the database
+    db: (integration) tests touching the database
-    e2e: non-db integration tests
+    e2e: non-db and non-r integration tests
+    r: (integration) tests using rpy2

src/urban_meal_delivery/configuration.py

@@ -69,6 +69,8 @@ class Config:
     ALEMBIC_TABLE = 'alembic_version'
     ALEMBIC_TABLE_SCHEMA = 'public'
 
+    R_LIBS_PATH = os.getenv('R_LIBS')
+
     def __repr__(self) -> str:
         """Non-literal text representation."""
         return '<configuration>'
@@ -117,6 +119,12 @@ def make_config(env: str = 'production') -> Config:
     if config.DATABASE_URI is None and not os.getenv('TESTING'):
         warnings.warn('Bad configurartion: no DATABASE_URI set in the environment')
 
+    # Some functionalities require R and some packages installed.
+    # To ensure isolation and reproducibility, the projects keeps the R dependencies
+    # in a project-local folder that must be set in the environment.
+    if config.R_LIBS_PATH is None and not os.getenv('TESTING'):
+        warnings.warn('Bad configuration: no R_LIBS set in the environment')
+
     return config
 
 

src/urban_meal_delivery/init_r.py

@@ -0,0 +1,28 @@
+"""Initialize the R dependencies.
+
+The purpose of this module is to import all the R packages that are installed
+into a sub-folder (see `config.R_LIBS_PATH`) in the project's root directory.
+
+The Jupyter notebook "research/r_dependencies.ipynb" can be used to install all
+R dependencies on a Ubuntu/Debian based system.
+"""
+
+from rpy2.rinterface_lib import callbacks as rcallbacks
+from rpy2.robjects import packages as rpackages
+
+
+# Suppress R's messages to stdout and stderr.
+# Source: https://stackoverflow.com/a/63220287
+rcallbacks.consolewrite_print = lambda msg: None  # pragma: no cover
+rcallbacks.consolewrite_warnerror = lambda msg: None  # pragma: no cover
+
+
+# For clarity and convenience, re-raise the error that results from missing R
+# dependencies with clearer instructions as to how to deal with it.
+try:  # noqa:WPS229
+    rpackages.importr('forecast')
+    rpackages.importr('zoo')
+
+except rpackages.PackageNotInstalledError:  # pragma: no cover
+    msg = 'See the "research/r_dependencies.ipynb" notebook!'
+    raise rpackages.PackageNotInstalledError(msg) from None

tests/test_config.py

@@ -29,6 +29,9 @@ def test_database_uri_set(env, monkeypatch):
     monkeypatch.setattr(configuration.ProductionConfig, 'DATABASE_URI', uri)
     monkeypatch.setattr(configuration.TestingConfig, 'DATABASE_URI', uri)
 
+    # Prevent that a warning is emitted for a missing R_LIBS_PATH.
+    monkeypatch.setattr(configuration.Config, 'R_LIBS_PATH', '.cache/r_libs')
+
     with pytest.warns(None) as record:
         configuration.make_config(env)
 
@@ -43,6 +46,9 @@ def test_no_database_uri_set_with_testing_env_var(env, monkeypatch):
 
     monkeypatch.setenv('TESTING', 'true')
 
+    # Prevent that a warning is emitted for a missing R_LIBS_PATH.
+    monkeypatch.setattr(configuration.Config, 'R_LIBS_PATH', '.cache/r_libs')
+
     with pytest.warns(None) as record:
         configuration.make_config(env)
 
@@ -57,10 +63,64 @@ def test_no_database_uri_set_without_testing_env_var(env, monkeypatch):
 
     monkeypatch.delenv('TESTING', raising=False)
 
+    # Prevent that a warning is emitted for a missing R_LIBS_PATH.
+    monkeypatch.setattr(configuration.Config, 'R_LIBS_PATH', '.cache/r_libs')
+
     with pytest.warns(UserWarning, match='no DATABASE_URI'):
         configuration.make_config(env)
 
 
+@pytest.mark.parametrize('env', envs)
+def test_r_libs_path_set(env, monkeypatch):
+    """Package does NOT emit a warning if R_LIBS is set in the environment."""
+    monkeypatch.setattr(configuration.Config, 'R_LIBS_PATH', '.cache/r_libs')
+
+    # Prevent that a warning is emitted for a missing DATABASE_URI.
+    uri = 'postgresql://user:password@localhost/db'
+    monkeypatch.setattr(configuration.ProductionConfig, 'DATABASE_URI', uri)
+
+    with pytest.warns(None) as record:
+        configuration.make_config(env)
+
+    assert len(record) == 0  # noqa:WPS441,WPS507
+
+
+@pytest.mark.parametrize('env', envs)
+def test_no_r_libs_path_set_with_testing_env_var(env, monkeypatch):
+    """Package emits a warning if no R_LIBS is set in the environment ...
+
+    ... when not testing.
+    """
+    monkeypatch.setattr(configuration.Config, 'R_LIBS_PATH', None)
+    monkeypatch.setenv('TESTING', 'true')
+
+    # Prevent that a warning is emitted for a missing DATABASE_URI.
+    uri = 'postgresql://user:password@localhost/db'
+    monkeypatch.setattr(configuration.ProductionConfig, 'DATABASE_URI', uri)
+
+    with pytest.warns(None) as record:
+        configuration.make_config(env)
+
+    assert len(record) == 0  # noqa:WPS441,WPS507
+
+
+@pytest.mark.parametrize('env', envs)
+def test_no_r_libs_path_set_without_testing_env_var(env, monkeypatch):
+    """Package emits a warning if no R_LIBS is set in the environment ...
+
+    ... when not testing.
+    """
+    monkeypatch.setattr(configuration.Config, 'R_LIBS_PATH', None)
+    monkeypatch.delenv('TESTING', raising=False)
+
+    # Prevent that a warning is emitted for a missing DATABASE_URI.
+    uri = 'postgresql://user:password@localhost/db'
+    monkeypatch.setattr(configuration.ProductionConfig, 'DATABASE_URI', uri)
+
+    with pytest.warns(UserWarning, match='no R_LIBS'):
+        configuration.make_config(env)
+
+
 def test_random_testing_schema():
     """CLEAN_SCHEMA is randomized if not set explicitly."""
     result = configuration.random_schema_name()

tests/test_init_r.py

@@ -0,0 +1,19 @@
+"""Verify that the R packages are installed correctly."""
+
+import pytest
+
+
+@pytest.mark.r
+def test_r_packages_installed():
+    """Import the `urban_meal_delivery.init_r` module.
+
+    Doing this raises a `PackageNotInstalledError` if the
+    mentioned R packages are not importable.
+
+    They must be installed externally. That happens either
+    in the "research/r_dependencies.ipynb" notebook or
+    in the GitHub Actions CI.
+    """
+    from urban_meal_delivery import init_r  # noqa:WPS433
+
+    assert init_r is not None