Add CLI script to gridify all cities

- reorganize `urban_meal_delivery.console` into a sub-package
- move `tests.db.conftest` fixtures into `tests.conftest`
  => some integration tests regarding CLI scripts need a database
- add `urban_meal_delivery.console.decorators.db_revision` decorator
  to ensure the database is at a certain state before a CLI script runs
- refactor the `urban_meal_delivery.db.grids.Grid.gridify()` constructor:
  - bug fix: even empty `Pixel`s end up in the database temporarily
    => create `Pixel` objects only if an `Address` is to be assigned
       to it
  - streamline code and docstring
  - add further test cases
This commit is contained in:
Alexander Hess 2021-01-06 16:17:05 +01:00
parent daa224d041
commit 54ff377579
Signed by: alexander
GPG key ID: 344EA5AB10D868E0
15 changed files with 372 additions and 160 deletions

View file

@ -88,4 +88,4 @@ sphinx = "^3.1.2"
sphinx-autodoc-typehints = "^1.11.0"
[tool.poetry.scripts]
umd = "urban_meal_delivery.console:main"
umd = "urban_meal_delivery.console:cli"

View file

@ -32,6 +32,8 @@ class Config:
# time horizon, we treat it as an ad-hoc order.
QUASI_AD_HOC_LIMIT = datetime.timedelta(minutes=45)
GRID_SIDE_LENGTHS = [707, 1000, 1414]
DATABASE_URI = os.getenv('DATABASE_URI')
# The PostgreSQL schema that holds the tables with the original data.

View file

@ -0,0 +1,9 @@
"""Provide CLI scripts for the project."""
from urban_meal_delivery.console import gridify
from urban_meal_delivery.console import main
cli = main.entry_point
cli.add_command(gridify.gridify)

View file

@ -0,0 +1,37 @@
"""Utils for the CLI scripts."""
import functools
import os
import subprocess # noqa:S404
import sys
from typing import Any, Callable
import click
def db_revision(rev: str) -> Callable: # pragma: no cover -> easy to check visually
"""A decorator ensuring the database is at a given revision."""
def decorator(func: Callable) -> Callable:
@functools.wraps(func)
def ensure(*args: Any, **kwargs: Any) -> Any: # noqa:WPS430
"""Do not execute the `func` if the revision does not match."""
if not os.getenv('TESTING'):
result = subprocess.run( # noqa:S603,S607
['alembic', 'current'],
capture_output=True,
check=False,
encoding='utf8',
)
if not result.stdout.startswith(rev):
click.echo(
click.style(f'Database is not at revision {rev}', fg='red'),
)
sys.exit(1)
return func(*args, **kwargs)
return ensure
return decorator

View file

@ -0,0 +1,48 @@
"""CLI script to create pixel grids."""
import click
from urban_meal_delivery import config
from urban_meal_delivery import db
from urban_meal_delivery.console import decorators
@click.command()
@decorators.db_revision('888e352d7526')
def gridify() -> None: # pragma: no cover note:b1f68d24
"""Create grids for all cities.
This command creates grids with pixels of various
side lengths (specified in `urban_meal_delivery.config`).
Pixels are only generated if they contain at least one
(pickup or delivery) address.
All data are persisted to the database.
"""
cities = db.session.query(db.City).all()
click.echo(f'{len(cities)} cities retrieved from the database')
for city in cities:
click.echo(f'\nCreating grids for {city.name}')
for side_length in config.GRID_SIDE_LENGTHS:
click.echo(f'Creating grid with a side length of {side_length} meters')
grid = db.Grid.gridify(city=city, side_length=side_length)
db.session.add(grid)
click.echo(f' -> created {len(grid.pixels)} pixels')
# The number of assigned addresses is the same across different `side_length`s.
db.session.flush() # necessary for the query to work
n_assigned = (
db.session.query(db.AddressPixelAssociation)
.filter(db.AddressPixelAssociation.grid_id == grid.id)
.count()
)
click.echo(
f'=> assigned {n_assigned} out of {len(city.addresses)} addresses in {city.name}', # noqa:E501
)
db.session.commit()

View file

@ -1,14 +1,14 @@
"""Provide CLI scripts for the project."""
"""The entry point for all CLI scripts in the project."""
from typing import Any
import click
from click.core import Context
from click import core as cli_core
import urban_meal_delivery
def show_version(ctx: Context, _param: Any, value: bool) -> None:
def show_version(ctx: cli_core.Context, _param: Any, value: bool) -> None:
"""Show the package's version."""
# If --version / -V is NOT passed in,
# continue with the command.
@ -24,7 +24,7 @@ def show_version(ctx: Context, _param: Any, value: bool) -> None:
ctx.exit()
@click.command()
@click.group()
@click.option(
'--version',
'-V',
@ -33,5 +33,5 @@ def show_version(ctx: Context, _param: Any, value: bool) -> None:
is_eager=True,
expose_value=False,
)
def main() -> None:
def entry_point() -> None:
"""The urban-meal-delivery research project."""

View file

@ -10,15 +10,17 @@ That is the case on the CI server.
import os
import sqlalchemy as sa
from sqlalchemy import engine as engine_mod
from sqlalchemy import orm
import urban_meal_delivery
if os.getenv('TESTING'):
engine = None
connection = None
session = None
# Specify the types explicitly to make mypy happy.
engine: engine_mod.Engine = None
connection: engine_mod.Connection = None
session: orm.Session = None
else: # pragma: no cover
engine = sa.create_engine(urban_meal_delivery.config.DATABASE_URI)

View file

@ -57,8 +57,8 @@ class Grid(meta.Base):
def gridify(cls, city: db.City, side_length: int) -> db.Grid:
"""Create a fully populated `Grid` for a `city`.
The created `Grid` contains only the `Pixel`s for which
there is at least one `Address` in it.
The `Grid` contains only `Pixel`s that have at least one `Address`.
`Address` objects outside the `city`'s viewport are discarded.
Args:
city: city for which the grid is created
@ -69,28 +69,30 @@ class Grid(meta.Base):
"""
grid = cls(city=city, side_length=side_length)
# Create `Pixel` objects covering the entire `city`.
# Note: `+1` so that `city.northeast` corner is on the grid.
possible_pixels = [
db.Pixel(n_x=n_x, n_y=n_y)
for n_x in range((city.total_x // side_length) + 1)
for n_y in range((city.total_y // side_length) + 1)
]
# For convenient lookup by `.n_x`-`.n_y` coordinates.
pixel_map = {(pixel.n_x, pixel.n_y): pixel for pixel in possible_pixels}
# `Pixel`s grouped by `.n_x`-`.n_y` coordinates.
pixels = {}
for address in city.addresses:
# Determine which `pixel` the `address` belongs to.
n_x = address.x // side_length
n_y = address.y // side_length
pixel = pixel_map[n_x, n_y]
# Check if an `address` is not within the `city`'s viewport, ...
not_within_city_viewport = (
address.x < 0
or address.x > city.total_x
or address.y < 0
or address.y > city.total_y
)
# ... and, if so, the `address` does not belong to any `Pixel`.
if not_within_city_viewport:
continue
# Determine which `pixel` the `address` belongs to ...
n_x, n_y = address.x // side_length, address.y // side_length
# ... and create a new `Pixel` object if necessary.
if (n_x, n_y) not in pixels:
pixels[(n_x, n_y)] = db.Pixel(grid=grid, n_x=n_x, n_y=n_y)
pixel = pixels[(n_x, n_y)]
# Create an association between the `address` and `pixel`.
assoc = db.AddressPixelAssociation(address=address, pixel=pixel)
pixel.addresses.append(assoc)
# Only keep `pixel`s that contain at least one `Address`.
grid.pixels = [pixel for pixel in pixel_map.values() if pixel.addresses]
return grid

View file

@ -1,8 +1,20 @@
"""Utils for testing the entire package."""
"""Fixtures for testing the entire package.
The ORM related fixtures are placed here too as some integration tests
in the CLI layer need access to the database.
"""
import os
import pytest
import sqlalchemy as sa
from alembic import command as migrations_cmd
from alembic import config as migrations_config
from sqlalchemy import orm
from tests.db import fake_data
from urban_meal_delivery import config
from urban_meal_delivery import db
# The TESTING environment variable is set
@ -12,3 +24,93 @@ if not os.getenv('TESTING'):
if not config.TESTING:
raise RuntimeError('The testing configuration was not loaded')
@pytest.fixture(scope='session', params=['all_at_once', 'sequentially'])
def db_connection(request):
"""Create all tables given the ORM models.
The tables are put into a distinct PostgreSQL schema
that is removed after all tests are over.
The database connection used to do that is yielded.
There are two modes for this fixture:
- "all_at_once": build up the tables all at once with MetaData.create_all()
- "sequentially": build up the tables sequentially with `alembic upgrade head`
This ensures that Alembic's migration files are consistent.
"""
# We need a fresh database connection for each of the two `params`.
# Otherwise, the first test of the parameter run second will fail.
engine = sa.create_engine(config.DATABASE_URI)
connection = engine.connect()
# Monkey patch the package's global `engine` and `connection` objects,
# just in case if it is used somewhere in the code base.
db.engine = engine
db.connection = connection
if request.param == 'all_at_once':
connection.execute(f'CREATE SCHEMA {config.CLEAN_SCHEMA};')
db.Base.metadata.create_all(connection)
else:
cfg = migrations_config.Config('alembic.ini')
migrations_cmd.upgrade(cfg, 'head')
try:
yield connection
finally:
connection.execute(f'DROP SCHEMA {config.CLEAN_SCHEMA} CASCADE;')
if request.param == 'sequentially':
tmp_alembic_version = f'{config.ALEMBIC_TABLE}_{config.CLEAN_SCHEMA}'
connection.execute(
f'DROP TABLE {config.ALEMBIC_TABLE_SCHEMA}.{tmp_alembic_version};',
)
connection.close()
@pytest.fixture
def db_session(db_connection):
"""A SQLAlchemy session that rolls back everything after a test case."""
# Begin the outermost transaction
# that is rolled back at the end of each test case.
transaction = db_connection.begin()
# Create a session bound to the same connection as the `transaction`.
# Using any other session would not result in the roll back.
session = orm.sessionmaker()(bind=db_connection)
# Monkey patch the package's global `session` object,
# which is used heavily in the code base.
db.session = session
try:
yield session
finally:
session.close()
transaction.rollback()
# Import the fixtures from the `fake_data` sub-package.
make_address = fake_data.make_address
make_courier = fake_data.make_courier
make_customer = fake_data.make_customer
make_order = fake_data.make_order
make_restaurant = fake_data.make_restaurant
address = fake_data.address
city = fake_data.city
city_data = fake_data.city_data
courier = fake_data.courier
customer = fake_data.customer
order = fake_data.order
restaurant = fake_data.restaurant
grid = fake_data.grid
pixel = fake_data.pixel

View file

@ -0,0 +1,5 @@
"""Test the CLI scripts in the urban-meal-delivery package.
Some tests require a database. Therefore, the corresponding code is excluded
from coverage reporting with "pragma: no cover" (grep:b1f68d24).
"""

10
tests/console/conftest.py Normal file
View file

@ -0,0 +1,10 @@
"""Fixture for testing the CLI scripts."""
import pytest
from click import testing as click_testing
@pytest.fixture
def cli() -> click_testing.CliRunner:
"""Initialize Click's CLI Test Runner."""
return click_testing.CliRunner()

View file

@ -0,0 +1,41 @@
"""Integration test for the `urban_meal_delivery.console.gridify` module."""
import pytest
import urban_meal_delivery
from urban_meal_delivery import db
from urban_meal_delivery.console import gridify
@pytest.mark.db
def test_four_pixels_with_two_addresses(
cli, db_session, monkeypatch, city, make_address,
):
"""Two `Address` objects in distinct `Pixel` objects.
This is roughly the same test case as
`tests.db.test_grids.test_four_pixels_with_two_addresses`.
The difference is that the result is written to the database.
"""
# Create two `Address` objects in distinct `Pixel`s.
city.addresses = [
# One `Address` in the lower-left `Pixel`, ...
make_address(latitude=48.8357377, longitude=2.2517412),
# ... and another one in the upper-right one.
make_address(latitude=48.8898312, longitude=2.4357622),
]
db_session.add(city)
db_session.commit()
side_length = max(city.total_x // 2, city.total_y // 2) + 1
# Hack the configuration regarding the grids to be created.
monkeypatch.setattr(urban_meal_delivery.config, 'GRID_SIDE_LENGTHS', [side_length])
result = cli.invoke(gridify.gridify)
assert result.exit_code == 0
assert db_session.query(db.Grid).count() == 1
assert db_session.query(db.Pixel).count() == 2

View file

@ -1,18 +1,17 @@
"""Test the package's `umd` command-line client."""
"""Test the package's top-level `umd` CLI command."""
import click
import pytest
from click import testing as click_testing
from urban_meal_delivery import console
from urban_meal_delivery.console import main
class TestShowVersion:
"""Test console.show_version().
"""Test `console.main.show_version()`.
The function is used as a callback to a click command option.
show_version() prints the name and version of the installed package to
`show_version()` prints the name and version of the installed package to
stdout. The output looks like this: "{pkg_name}, version {version}".
Development (= non-final) versions are indicated by appending a
@ -23,12 +22,12 @@ class TestShowVersion:
@pytest.fixture
def ctx(self) -> click.Context:
"""Context around the console.main Command."""
return click.Context(console.main)
"""Context around the `main.entry_point` Command."""
return click.Context(main.entry_point)
def test_no_version(self, capsys, ctx):
"""The the early exit branch without any output."""
console.show_version(ctx, _param='discarded', value=False)
"""Test the early exit branch without any output."""
main.show_version(ctx, _param='discarded', value=False)
captured = capsys.readouterr()
@ -37,10 +36,10 @@ class TestShowVersion:
def test_final_version(self, capsys, ctx, monkeypatch):
"""For final versions, NO "development" warning is emitted."""
version = '1.2.3'
monkeypatch.setattr(console.urban_meal_delivery, '__version__', version)
monkeypatch.setattr(main.urban_meal_delivery, '__version__', version)
with pytest.raises(click.exceptions.Exit):
console.show_version(ctx, _param='discarded', value=True)
main.show_version(ctx, _param='discarded', value=True)
captured = capsys.readouterr()
@ -49,18 +48,18 @@ class TestShowVersion:
def test_develop_version(self, capsys, ctx, monkeypatch):
"""For develop versions, a warning thereof is emitted."""
version = '1.2.3.dev0'
monkeypatch.setattr(console.urban_meal_delivery, '__version__', version)
monkeypatch.setattr(main.urban_meal_delivery, '__version__', version)
with pytest.raises(click.exceptions.Exit):
console.show_version(ctx, _param='discarded', value=True)
main.show_version(ctx, _param='discarded', value=True)
captured = capsys.readouterr()
assert captured.out.strip().endswith(f', version {version} (development)')
class TestCLI:
"""Test the `umd` CLI utility.
class TestCLIWithoutCommand:
"""Test the `umd` CLI utility, invoked without any specific command.
The test cases are integration tests.
Therefore, they are not considered for coverage reporting.
@ -68,18 +67,12 @@ class TestCLI:
# pylint:disable=no-self-use
@pytest.fixture
def cli(self) -> click_testing.CliRunner:
"""Initialize Click's CLI Test Runner."""
return click_testing.CliRunner()
@pytest.mark.no_cover
def test_no_options(self, cli):
"""Exit with 0 status code and no output if run without options."""
result = cli.invoke(console.main)
result = cli.invoke(main.entry_point)
assert result.exit_code == 0
assert result.output == ''
# The following test cases validate the --version / -V option.
@ -90,9 +83,9 @@ class TestCLI:
def test_final_version(self, cli, monkeypatch, option):
"""For final versions, NO "development" warning is emitted."""
version = '1.2.3'
monkeypatch.setattr(console.urban_meal_delivery, '__version__', version)
monkeypatch.setattr(main.urban_meal_delivery, '__version__', version)
result = cli.invoke(console.main, option)
result = cli.invoke(main.entry_point, option)
assert result.exit_code == 0
assert result.output.strip().endswith(f', version {version}')
@ -102,9 +95,9 @@ class TestCLI:
def test_develop_version(self, cli, monkeypatch, option):
"""For develop versions, a warning thereof is emitted."""
version = '1.2.3.dev0'
monkeypatch.setattr(console.urban_meal_delivery, '__version__', version)
monkeypatch.setattr(main.urban_meal_delivery, '__version__', version)
result = cli.invoke(console.main, option)
result = cli.invoke(main.entry_point, option)
assert result.exit_code == 0
assert result.output.strip().endswith(f', version {version} (development)')

View file

@ -1,101 +0,0 @@
"""Utils for testing the ORM layer."""
import pytest
import sqlalchemy as sa
from alembic import command as migrations_cmd
from alembic import config as migrations_config
from sqlalchemy import orm
from tests.db import fake_data
from urban_meal_delivery import config
from urban_meal_delivery import db
@pytest.fixture(scope='session', params=['all_at_once', 'sequentially'])
def db_connection(request):
"""Create all tables given the ORM models.
The tables are put into a distinct PostgreSQL schema
that is removed after all tests are over.
The database connection used to do that is yielded.
There are two modes for this fixture:
- "all_at_once": build up the tables all at once with MetaData.create_all()
- "sequentially": build up the tables sequentially with `alembic upgrade head`
This ensures that Alembic's migration files are consistent.
"""
# We need a fresh database connection for each of the two `params`.
# Otherwise, the first test of the parameter run second will fail.
engine = sa.create_engine(config.DATABASE_URI)
connection = engine.connect()
# Monkey patch the package's global `engine` and `connection` objects,
# just in case if it is used somewhere in the code base.
db.engine = engine
db.connection = connection
if request.param == 'all_at_once':
connection.execute(f'CREATE SCHEMA {config.CLEAN_SCHEMA};')
db.Base.metadata.create_all(connection)
else:
cfg = migrations_config.Config('alembic.ini')
migrations_cmd.upgrade(cfg, 'head')
try:
yield connection
finally:
connection.execute(f'DROP SCHEMA {config.CLEAN_SCHEMA} CASCADE;')
if request.param == 'sequentially':
tmp_alembic_version = f'{config.ALEMBIC_TABLE}_{config.CLEAN_SCHEMA}'
connection.execute(
f'DROP TABLE {config.ALEMBIC_TABLE_SCHEMA}.{tmp_alembic_version};',
)
connection.close()
@pytest.fixture
def db_session(db_connection):
"""A SQLAlchemy session that rolls back everything after a test case."""
# Begin the outermost transaction
# that is rolled back at the end of each test case.
transaction = db_connection.begin()
# Create a session bound to the same connection as the `transaction`.
# Using any other session would not result in the roll back.
session = orm.sessionmaker()(bind=db_connection)
# Monkey patch the package's global `session` object,
# which is used heavily in the code base.
db.session = session
try:
yield session
finally:
session.close()
transaction.rollback()
# Import the fixtures from the `fake_data` sub-package.
make_address = fake_data.make_address
make_courier = fake_data.make_courier
make_customer = fake_data.make_customer
make_order = fake_data.make_order
make_restaurant = fake_data.make_restaurant
address = fake_data.address
city = fake_data.city
city_data = fake_data.city_data
courier = fake_data.courier
customer = fake_data.customer
order = fake_data.order
restaurant = fake_data.restaurant
grid = fake_data.grid
pixel = fake_data.pixel

View file

@ -75,11 +75,15 @@ class TestProperties:
class TestGridification:
"""Test the `Grid.gridify()` constructor."""
def test_one_pixel_covering_entire_city_without_addresses(self, city):
@pytest.mark.no_cover
def test_one_pixel_without_addresses(self, city):
"""At the very least, there must be one `Pixel` ...
... if the `side_length` is greater than both the
horizontal and vertical distances of the viewport.
This test case skips the `for`-loop inside `Grid.gridify()`.
Interestingly, coverage.py does not see this.
"""
city.addresses = []
@ -91,7 +95,7 @@ class TestGridification:
assert isinstance(result, db.Grid)
assert len(result.pixels) == 0 # noqa:WPS507
def test_one_pixel_covering_entire_city_with_one_address(self, city, address):
def test_one_pixel_with_one_address(self, city, address):
"""At the very least, there must be one `Pixel` ...
... if the `side_length` is greater than both the
@ -107,8 +111,66 @@ class TestGridification:
assert isinstance(result, db.Grid)
assert len(result.pixels) == 1
def test_one_pixel_with_two_addresses(self, city, make_address):
"""At the very least, there must be one `Pixel` ...
... if the `side_length` is greater than both the
horizontal and vertical distances of the viewport.
This test case is necessary as `test_one_pixel_with_one_address`
does not have to re-use an already created `Pixel` object internally.
"""
city.addresses = [make_address(), make_address()]
# `+1` as otherwise there would be a second pixel in one direction.
side_length = max(city.total_x, city.total_y) + 1
result = db.Grid.gridify(city=city, side_length=side_length)
assert isinstance(result, db.Grid)
assert len(result.pixels) == 1
def test_one_pixel_with_address_too_far_south(self, city, address):
"""An `address` outside the `city`'s viewport is discarded."""
# Move the `address` just below `city.southwest`.
address.latitude = city.southwest.latitude - 0.1
city.addresses = [address]
# `+1` as otherwise there would be a second pixel in one direction.
side_length = max(city.total_x, city.total_y) + 1
result = db.Grid.gridify(city=city, side_length=side_length)
assert isinstance(result, db.Grid)
assert len(result.pixels) == 0 # noqa:WPS507
@pytest.mark.no_cover
def test_one_pixel_with_address_too_far_west(self, city, address):
"""An `address` outside the `city`'s viewport is discarded.
This test is a logical sibling to `test_one_pixel_with_address_too_far_south`
and therefore redundant.
"""
# Move the `address` just left to `city.southwest`.
address.longitude = city.southwest.longitude - 0.1
city.addresses = [address]
# `+1` as otherwise there would be a second pixel in one direction.
side_length = max(city.total_x, city.total_y) + 1
result = db.Grid.gridify(city=city, side_length=side_length)
assert isinstance(result, db.Grid)
assert len(result.pixels) == 0 # noqa:WPS507
@pytest.mark.no_cover
def test_four_pixels_with_two_addresses(self, city, make_address):
"""Two `Address` objects in distinct `Pixel` objects."""
"""Two `Address` objects in distinct `Pixel` objects.
This test is more of a sanity check.
"""
# Create two `Address` objects in distinct `Pixel`s.
city.addresses = [
# One `Address` in the lower-left `Pixel`, ...
@ -136,7 +198,7 @@ class TestGridification:
def test_make_random_grids(self, db_session, city, make_address, side_length):
"""With 100 random `Address` objects, a grid must have ...
... between 1 and a deterministic number of `Pixel` objects.
... between 1 and a deterministic upper bound of `Pixel` objects.
This test creates confidence that the created `Grid`
objects adhere to the database constraints.