diff --git a/CHANGELOG.md b/CHANGELOG.md index f4371a4..a855c37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,19 +5,6 @@ All notable changes to WuttJamaican will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## v0.18.0 (2024-12-15) - -### Feat - -- add basic batch feature, data model and partial handler -- add basic db handler, for tracking counter values - -### Fix - -- add basic execution methods for batch handler -- add `render_date()`, `render_datetime()` methods for app handler -- add command for `wutta make-appdir` - ## v0.17.1 (2024-12-08) ### Fix diff --git a/docs/api/wuttjamaican.batch.rst b/docs/api/wuttjamaican.batch.rst deleted file mode 100644 index a51fd77..0000000 --- a/docs/api/wuttjamaican.batch.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttjamaican.batch`` -====================== - -.. automodule:: wuttjamaican.batch - :members: diff --git a/docs/api/wuttjamaican.db.handler.rst b/docs/api/wuttjamaican.db.handler.rst deleted file mode 100644 index 6ae56d3..0000000 --- a/docs/api/wuttjamaican.db.handler.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttjamaican.db.handler`` -=========================== - -.. automodule:: wuttjamaican.db.handler - :members: diff --git a/docs/api/wuttjamaican.db.model.batch.rst b/docs/api/wuttjamaican.db.model.batch.rst deleted file mode 100644 index 050c04d..0000000 --- a/docs/api/wuttjamaican.db.model.batch.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttjamaican.db.model.batch`` -=============================== - -.. automodule:: wuttjamaican.db.model.batch - :members: diff --git a/docs/conf.py b/docs/conf.py index 8dbfd79..56ce5da 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,6 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] intersphinx_mapping = { 'alembic': ('https://alembic.sqlalchemy.org/en/latest/', None), - 'humanize': ('https://humanize.readthedocs.io/en/stable/', None), 'mako': ('https://docs.makotemplates.org/en/latest/', None), 'packaging': ('https://packaging.python.org/en/latest/', None), 'python': ('https://docs.python.org/3/', None), diff --git a/docs/glossary.rst b/docs/glossary.rst index 58484a2..777ba0b 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -76,48 +76,6 @@ Glossary See also :class:`~wuttjamaican.auth.AuthHandler`. - batch - This refers to a process whereby bulk data operations may be - performed, with preview and other tools to allow the user to - refine as needed before "executing" the batch. - - The term "batch" may refer to such a feature overall, or the - :term:`data model` used, or the specific data for a single batch, - etc. - - See also :term:`batch handler` and :term:`batch row`, and the - :class:`~wuttjamaican.db.model.batch.BatchMixin` base class. - - batch handler - This refers to a :term:`handler` meant to process a given type of - :term:`batch`. - - There may be multiple handlers registered for a given - :term:`batch type`, but (usually) only one will be configured for - use. - - batch row - A row of data within a :term:`batch`. - - May also refer to the :term:`data model` class used for such a row. - - See also the :class:`~wuttjamaican.db.model.batch.BatchRowMixin` - base class. - - batch type - This term is used to distinguish :term:`batches ` according - to which underlying table is used to store their data, essentially. - - For instance a "pricing batch" would use one table, whereas an - "inventory batch" would use another. And each "type" would be - managed by its own :term:`batch handler`. - - The batch type is set on the model class but is also available on - the handler: - - * :attr:`wuttjamaican.db.model.batch.BatchMixin.batch_type` - * :attr:`wuttjamaican.batch.BatchHandler.batch_type` - command A top-level command line interface for the app. Note that top-level commands don't usually "do" anything per se, and are @@ -170,14 +128,6 @@ Glossary Most :term:`apps` will have at least one :term:`app database`. See also :doc:`narr/db/index`. - db handler - The :term:`handler` responsible for various operations involving - the :term:`app database` (and possibly other :term:`databases - `). - - See also the :class:`~wuttjamaican.db.handler.DatabaseHandler` - base class. - db session The "session" is a SQLAlchemy abstraction for an open database connection, essentially. diff --git a/docs/index.rst b/docs/index.rst index 896a543..cd2064f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,7 +64,6 @@ Contents api/wuttjamaican.app api/wuttjamaican.auth - api/wuttjamaican.batch api/wuttjamaican.cli api/wuttjamaican.cli.base api/wuttjamaican.cli.make_appdir @@ -72,11 +71,9 @@ Contents api/wuttjamaican.conf api/wuttjamaican.db api/wuttjamaican.db.conf - api/wuttjamaican.db.handler api/wuttjamaican.db.model api/wuttjamaican.db.model.auth api/wuttjamaican.db.model.base - api/wuttjamaican.db.model.batch api/wuttjamaican.db.model.upgrades api/wuttjamaican.db.sess api/wuttjamaican.db.util diff --git a/pyproject.toml b/pyproject.toml index bf74095..ec67103 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "WuttJamaican" -version = "0.18.0" +version = "0.17.1" description = "Base package for Wutta Framework" readme = "README.md" authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}] @@ -26,7 +26,6 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ - "humanize", 'importlib-metadata; python_version < "3.10"', "importlib_resources ; python_version < '3.9'", "Mako", diff --git a/src/wuttjamaican/app.py b/src/wuttjamaican/app.py index 102d5db..7a7000c 100644 --- a/src/wuttjamaican/app.py +++ b/src/wuttjamaican/app.py @@ -24,14 +24,11 @@ WuttJamaican - app handler """ -import datetime import importlib import os import sys import warnings -import humanize - from wuttjamaican.util import (load_entry_points, load_object, make_title, make_uuid, make_true_uuid, progress_loop, resource_path) @@ -84,7 +81,6 @@ class AppHandler: default_model_spec = 'wuttjamaican.db.model' default_enum_spec = 'wuttjamaican.enum' default_auth_handler_spec = 'wuttjamaican.auth:AuthHandler' - default_db_handler_spec = 'wuttjamaican.db.handler:DatabaseHandler' default_email_handler_spec = 'wuttjamaican.email:EmailHandler' default_install_handler_spec = 'wuttjamaican.install:InstallHandler' default_people_handler_spec = 'wuttjamaican.people:PeopleHandler' @@ -497,7 +493,7 @@ class AppHandler: def make_true_uuid(self): """ - Generate a new UUID value. + Generate a new v7 UUID value. By default this simply calls :func:`wuttjamaican.util.make_true_uuid()`. @@ -518,7 +514,7 @@ class AppHandler: def make_uuid(self): """ - Generate a new UUID value. + Generate a new v7 UUID value. By default this simply calls :func:`wuttjamaican.util.make_uuid()`. @@ -717,21 +713,6 @@ class AppHandler: if value is not None: return value.strftime(self.display_format_datetime) - def render_time_ago(self, value): - """ - Return a human-friendly string, indicating how long ago - something occurred. - - Default logic uses :func:`humanize:humanize.naturaltime()` for - the rendering. - - :param value: Instance of :class:`python:datetime.datetime` or - :class:`python:datetime.timedelta`. - - :returns: Text to display. - """ - return humanize.naturaltime(value) - ############################## # getters for other handlers ############################## @@ -749,19 +730,6 @@ class AppHandler: self.handlers['auth'] = factory(self.config, **kwargs) return self.handlers['auth'] - def get_db_handler(self, **kwargs): - """ - Get the configured :term:`db handler`. - - :rtype: :class:`~wuttjamaican.db.handler.DatabaseHandler` - """ - if 'db' not in self.handlers: - spec = self.config.get(f'{self.appname}.db.handler', - default=self.default_db_handler_spec) - factory = self.load_object(spec) - self.handlers['db'] = factory(self.config, **kwargs) - return self.handlers['db'] - def get_email_handler(self, **kwargs): """ Get the configured :term:`email handler`. diff --git a/src/wuttjamaican/batch.py b/src/wuttjamaican/batch.py deleted file mode 100644 index 98325ff..0000000 --- a/src/wuttjamaican/batch.py +++ /dev/null @@ -1,448 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttJamaican -- Base package for Wutta Framework -# Copyright © 2023-2024 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -Batch Handlers -""" - -import datetime -import os -import shutil - -from wuttjamaican.app import GenericHandler - - -class BatchHandler(GenericHandler): - """ - Base class and *partial* default implementation for :term:`batch - handlers `. - - This handler class "works as-is" but does not actually do - anything. Subclass must implement logic for various things as - needed, e.g.: - - * :attr:`model_class` - * :meth:`init_batch()` - * :meth:`should_populate()` - * :meth:`populate()` - * :meth:`refresh_row()` - """ - - @property - def model_class(self): - """ - Reference to the batch :term:`data model` class which this - batch handler is meant to work with. - - This is expected to be a subclass of - :class:`~wuttjamaican.db.model.batch.BatchMixin` (among other - classes). - - Subclass must define this; default is not implemented. - """ - raise NotImplementedError("You must set the 'model_class' attribute " - f"for class '{self.__class__.__name__}'") - - @property - def batch_type(self): - """ - Convenience property to return the :term:`batch type` which - the current handler is meant to process. - - This is effectively an alias to - :attr:`~wuttjamaican.db.model.batch.BatchMixin.batch_type`. - """ - return self.model_class.batch_type - - def make_batch(self, session, progress=None, **kwargs): - """ - Make and return a new batch (:attr:`model_class`) instance. - - This will create the new batch, and auto-assign its - :attr:`~wuttjamaican.db.model.batch.BatchMixin.id` value - (unless caller specifies it) by calling - :meth:`consume_batch_id()`. - - It then will call :meth:`init_batch()` to perform any custom - initialization needed. - - Therefore callers should use this ``make_batch()`` method, but - subclass should override :meth:`init_batch()` instead (if - needed). - - :param session: Current :term:`db session`. - - :param progress: Optional progress indicator factory. - - :param \**kwargs: Additional kwargs to pass to the batch - constructor. - - :returns: New batch; instance of :attr:`model_class`. - """ - # generate new ID unless caller specifies - if 'id' not in kwargs: - kwargs['id'] = self.consume_batch_id(session) - - # make batch - batch = self.model_class(**kwargs) - self.init_batch(batch, session=session, progress=progress, **kwargs) - return batch - - def consume_batch_id(self, session, as_str=False): - """ - Fetch a new batch ID from the counter, and return it. - - This may be called automatically from :meth:`make_batch()`. - - :param session: Current :term:`db session`. - - :param as_str: Indicates the return value should be a string - instead of integer. - - :returns: Batch ID as integer, or zero-padded 8-char string. - """ - db = self.app.get_db_handler() - batch_id = db.next_counter_value(session, 'batch_id') - if as_str: - return f'{batch_id:08d}' - return batch_id - - def init_batch(self, batch, session=None, progress=None, **kwargs): - """ - Initialize a new batch. - - This is called automatically from :meth:`make_batch()`. - - Default logic does nothing; subclass should override if needed. - - .. note:: - *Population* of the new batch should **not** happen here; - see instead :meth:`populate()`. - """ - - def get_data_path(self, batch=None, filename=None, makedirs=False): - """ - Returns a path to batch data file(s). - - This can be used to return any of the following, depending on - how it's called: - - * path to root data dir for handler's :attr:`batch_type` - * path to data dir for specific batch - * path to specific filename, for specific batch - - For instance:: - - # nb. assuming batch_type = 'inventory' - batch = handler.make_batch(session, created_by=user) - - handler.get_data_path() - # => env/app/data/batch/inventory - - handler.get_data_path(batch) - # => env/app/data/batch/inventory/03/7721fe56c811ef9223743af49773a4 - - handler.get_data_path(batch, 'counts.csv') - # => env/app/data/batch/inventory/03/7721fe56c811ef9223743af49773a4/counts.csv - - :param batch: Optional batch instance. If specified, will - return path for this batch in particular. Otherwise will - return the "generic" path for handler's batch type. - - :param filename: Optional filename, in context of the batch. - If set, the returned path will include this filename. Only - relevant if ``batch`` is also specified. - - :param makedirs: Whether the folder(s) should be created, if - not already present. - - :returns: Path to root data dir for handler's batch type. - """ - # get root storage path - rootdir = self.config.get(f'{self.config.appname}.batch.storage_path') - if not rootdir: - appdir = self.app.get_appdir() - rootdir = os.path.join(appdir, 'data', 'batch') - - # get path for this batch type - path = os.path.join(rootdir, self.batch_type) - - # give more precise path, if batch was specified - if batch: - uuid = batch.uuid.hex - # nb. we use *last 2 chars* for first part of batch uuid - # path. this is because uuid7 is mostly sequential, so - # first 2 chars do not vary enough. - path = os.path.join(path, uuid[-2:], uuid[:-2]) - - # maybe create data dir - if makedirs and not os.path.exists(path): - os.makedirs(path) - - # append filename if applicable - if batch and filename: - path = os.path.join(path, filename) - - return path - - def should_populate(self, batch): - """ - Must return true or false, indicating whether the given batch - should be populated from initial data source(s). - - So, true means fill the batch with data up front - by calling - :meth:`do_populate()` - and false means the batch will start - empty. - - Default logic here always return false; subclass should - override if needed. - """ - return False - - def do_populate(self, batch, progress=None): - """ - Populate the batch from initial data source(s). - - This method is a convenience wrapper, which ultimately will - call :meth:`populate()` for the implementation logic. - - Therefore callers should use this ``do_populate()`` method, - but subclass should override :meth:`populate()` instead (if - needed). - - See also :meth:`should_populate()` - you should check that - before calling ``do_populate()``. - """ - self.populate(batch, progress=progress) - - def populate(self, batch, progress=None): - """ - Populate the batch from initial data source(s). - - It is assumed that the data source(s) to be used will be known - by inspecting various properties of the batch itself. - - Subclass should override this method to provide the - implementation logic. It may populate some batches - differently based on the batch attributes, or it may populate - them all the same. Whatever is needed. - - Callers should always use :meth:`do_populate()` instead of - calling ``populate()`` directly. - """ - - def make_row(self, **kwargs): - """ - Make a new row for the batch. This will be an instance of - :attr:`~wuttjamaican.db.model.batch.BatchMixin.__row_class__`. - - Note that the row will **not** be added to the batch; that - should be done with :meth:`add_row()`. - - :returns: A new row object, which does *not* yet belong to any batch. - """ - return self.model_class.__row_class__(**kwargs) - - def add_row(self, batch, row): - """ - Add the given row to the given batch. - - This assumes a *new* row which does not yet belong to a batch, - as returned by :meth:`make_row()`. - - It will add it to batch - :attr:`~wuttjamaican.db.model.batch.BatchMixin.rows`, call - :meth:`refresh_row()` for it, and update the - :attr:`~wuttjamaican.db.model.batch.BatchMixin.row_count`. - """ - session = self.app.get_session(batch) - with session.no_autoflush: - batch.rows.append(row) - self.refresh_row(row) - batch.row_count = (batch.row_count or 0) + 1 - - def refresh_row(self, row): - """ - Update the given batch row as needed, to reflect latest data. - - This method is a bit of a catch-all in that it could be used - to do any of the following (etc.): - - * fetch latest "live" data for comparison with batch input data - * (re-)calculate row values based on latest data - * set row status based on other row attributes - - This method is called when the row is first added to the batch - via :meth:`add_row()` - but may be called multiple times after - that depending on the workflow. - """ - - def why_not_execute(self, batch, user=None, **kwargs): - """ - Returns text indicating the reason (if any) that a given batch - should *not* be executed. - - By default the only reason a batch cannot be executed, is if - it has already been executed. But in some cases it should be - more restrictive; hence this method. - - A "brief but descriptive" message should be returned, which - may be displayed to the user e.g. so they understand why the - execute feature is not allowed for the batch. (There is no - need to check if batch is already executed since other logic - handles that.) - - If no text is returned, the assumption will be made that this - batch is safe to execute. - - :param batch: The batch in question; potentially eligible for - execution. - - :param user: :class:`~wuttjamaican.db.model.auth.User` who - might choose to execute the batch. - - :param \**kwargs: Execution kwargs for the batch, if known. - Should be similar to those for :meth:`execute()`. - - :returns: Text reason to prevent execution, or ``None``. - - The user interface should normally check this and if it - returns anything, that should be shown and the user should be - prevented from executing the batch. - - However :meth:`do_execute()` will also call this method, and - raise a ``RuntimeError`` if text was returned. This is done - out of safety, to avoid relying on the user interface. - """ - - def describe_execution(self, batch, user=None, **kwargs): - """ - This should return some text which briefly describes what will - happen when the given batch is executed. - - Note that Markdown is supported here, e.g.:: - - def describe_execution(self, batch, **kwargs): - return \""" - - This batch does some crazy things! - - **you cannot possibly fathom it** - - here are a few of them: - - - first - - second - - third - \""" - - Nothing is returned by default; subclass should define. - - :param batch: The batch in question; eligible for execution. - - :param user: Reference to current user who might choose to - execute the batch. - - :param \**kwargs: Execution kwargs for the batch; should be - similar to those for :meth:`execute()`. - - :returns: Markdown text describing batch execution. - """ - - def do_execute(self, batch, user, progress=None, **kwargs): - """ - Perform the execution steps for a batch. - - This first calls :meth:`why_not_execute()` to make sure this - is even allowed. - - If so, it calls :meth:`execute()` and then updates - :attr:`~wuttjamaican.db.model.batch.BatchMixin.executed` and - :attr:`~wuttjamaican.db.model.batch.BatchMixin.executed_by` on - the batch, to reflect current time+user. - - So, callers should use ``do_execute()``, and subclass should - override :meth:`execute()`. - - :param batch: The :term:`batch` to execute; instance of - :class:`~wuttjamaican.db.model.batch.BatchMixin` (among - other classes). - - :param user: :class:`~wuttjamaican.db.model.auth.User` who is - executing the batch. - - :param progress: Optional progress indicator factory. - - :param \**kwargs: Additional kwargs as needed. These are - passed as-is to :meth:`why_not_execute()` and - :meth:`execute()`. - """ - if batch.executed: - raise ValueError(f"batch has already been executed: {batch}") - - reason = self.why_not_execute(batch, user=user, **kwargs) - if reason: - raise RuntimeError(f"batch execution not allowed: {reason}") - - self.execute(batch, user=user, progress=progress, **kwargs) - batch.executed = datetime.datetime.now() - batch.executed_by = user - - def execute(self, batch, user=None, progress=None, **kwargs): - """ - Execute the given batch. - - Callers should use :meth:`do_execute()` instead, which calls - this method automatically. - - This does nothing by default; subclass must define logic. - - :param batch: A :term:`batch`; instance of - :class:`~wuttjamaican.db.model.batch.BatchMixin` (among - other classes). - - :param user: :class:`~wuttjamaican.db.model.auth.User` who is - executing the batch. - - :param progress: Optional progress indicator factory. - - :param \**kwargs: Additional kwargs which may affect the batch - execution behavior. There are none by default, but some - handlers may declare/use them. - """ - - def do_delete(self, batch, user, dry_run=False, progress=None, **kwargs): - """ - Delete the given batch entirely. - - This will delete the batch proper, all data rows, and any - files which may be associated with it. - """ - session = self.app.get_session(batch) - - # remove data files - path = self.get_data_path(batch) - if os.path.exists(path) and not dry_run: - shutil.rmtree(path) - - # remove batch proper - session.delete(batch) diff --git a/src/wuttjamaican/db/handler.py b/src/wuttjamaican/db/handler.py deleted file mode 100644 index 7c745d8..0000000 --- a/src/wuttjamaican/db/handler.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttJamaican -- Base package for Wutta Framework -# Copyright © 2024 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -Database Handler -""" - -import sqlalchemy as sa - -from wuttjamaican.app import GenericHandler - - -class DatabaseHandler(GenericHandler): - """ - Base class and default implementation for the :term:`db handler`. - """ - - def next_counter_value(self, session, key): - """ - Return the next counter value for the given key. - - If the DB backend is PostgreSQL, then a proper "sequence" is - used for the counter. - - All other backends use a "fake" sequence by creating a - dedicated table with auto-increment primary key, to provide - the counter. - - :param session: Current :term:`db session`. - - :param key: Unique key indicating the counter for which the - next value should be fetched. - - :returns: Next value as integer. - """ - dialect = session.bind.url.get_dialect().name - - # postgres uses "true" native sequence - if dialect == 'postgresql': - sql = f"create sequence if not exists {key}_seq" - session.execute(sa.text(sql)) - sql = f"select nextval('{key}_seq')" - value = session.execute(sa.text(sql)).scalar() - return value - - # otherwise use "magic" workaround - engine = session.bind - metadata = sa.MetaData() - table = sa.Table(f'_counter_{key}', metadata, - sa.Column('value', sa.Integer(), primary_key=True)) - table.create(engine, checkfirst=True) - with engine.begin() as cxn: - result = cxn.execute(table.insert()) - return result.lastrowid diff --git a/src/wuttjamaican/db/model/__init__.py b/src/wuttjamaican/db/model/__init__.py index 6a9883b..b0c5255 100644 --- a/src/wuttjamaican/db/model/__init__.py +++ b/src/wuttjamaican/db/model/__init__.py @@ -30,7 +30,6 @@ This namespace exposes the following: * :class:`~wuttjamaican.db.model.base.Base` * :func:`~wuttjamaican.db.util.uuid_column()` * :func:`~wuttjamaican.db.util.uuid_fk_column()` -* :class:`~wuttjamaican.db.util.UUID` And the :term:`data models `: @@ -41,16 +40,10 @@ And the :term:`data models `: * :class:`~wuttjamaican.db.model.auth.User` * :class:`~wuttjamaican.db.model.auth.UserRole` * :class:`~wuttjamaican.db.model.upgrades.Upgrade` - -And the :term:`batch` model base/mixin classes: - -* :class:`~wuttjamaican.db.model.batch.BatchMixin` -* :class:`~wuttjamaican.db.model.batch.BatchRowMixin` """ -from wuttjamaican.db.util import uuid_column, uuid_fk_column, UUID +from wuttjamaican.db.util import uuid_column, uuid_fk_column from .base import Base, Setting, Person from .auth import Role, Permission, User, UserRole from .upgrades import Upgrade -from .batch import BatchMixin, BatchRowMixin diff --git a/src/wuttjamaican/db/model/batch.py b/src/wuttjamaican/db/model/batch.py deleted file mode 100644 index f6ba154..0000000 --- a/src/wuttjamaican/db/model/batch.py +++ /dev/null @@ -1,423 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttJamaican -- Base package for Wutta Framework -# Copyright © 2023-2024 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -Batch data models -""" - -import datetime - -import sqlalchemy as sa -from sqlalchemy import orm -from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.ext.orderinglist import ordering_list - -from wuttjamaican.db.model import uuid_column, uuid_fk_column, User -from wuttjamaican.db.util import UUID - - -class BatchMixin: - """ - Mixin base class for :term:`data models ` which - represent a :term:`batch`. - - See also :class:`BatchRowMixin` which should be used for the row - model. - - For a batch model (table) to be useful, at least one :term:`batch - handler` must be defined, which is able to process data for that - :term:`batch type`. - - .. attribute:: batch_type - - This is the canonical :term:`batch type` for the batch model. - - By default this will match the underlying table name for the - batch, but the model class can set it explicitly to override. - - .. attribute:: __row_class__ - - Reference to the specific :term:`data model` class used for the - :term:`batch rows `. - - This will be a subclass of :class:`BatchRowMixin` (among other - classes). - - When defining the batch model, you do not have to set this as - it will be assigned automatically based on - :attr:`BatchRowMixin.__batch_class__`. - - .. attribute:: id - - Numeric ID for the batch, unique across all batches (regardless - of type). - - See also :attr:`id_str`. - - .. attribute:: description - - Simple description for the batch. - - .. attribute:: notes - - Arbitrary notes for the batch. - - .. attribute:: rows - - List of data rows for the batch, aka. :term:`batch rows `. - - Each will be an instance of :class:`BatchRowMixin` (among other - base classes). - - .. attribute:: row_count - - Cached row count for the batch, i.e. how many :attr:`rows` it has. - - No guarantees perhaps, but this should ideally be accurate (it - ultimately depends on the :term:`batch handler` - implementation). - - .. attribute:: STATUS - - Dict of possible batch status codes and their human-readable - names. - - Each key will be a possible :attr:`status_code` and the - corresponding value will be the human-readable name. - - See also :attr:`status_text` for when more detail/subtlety is - needed. - - Typically each "key" (code) is also defined as its own - "constant" on the model class. For instance:: - - from collections import OrderedDict - from wuttjamaican.db import model - - class MyBatch(model.BatchMixin, model.Base): - \""" my custom batch \""" - - STATUS_INCOMPLETE = 1 - STATUS_EXECUTABLE = 2 - - STATUS = OrderedDict([ - (STATUS_INCOMPLETE, "incomplete"), - (STATUS_EXECUTABLE, "executable"), - ]) - - # TODO: column definitions... - - And in fact, the above status definition is the built-in - default. However it is expected for subclass to overwrite the - definition entirely (in similar fashion to above) when needed. - - .. note:: - There is not any built-in logic around these integer codes; - subclass can use any the developer prefers. - - Of course, once you define one, if any live batches use it, - you should not then change its fundamental meaning (although - you can change the human-readable text). - - It's recommended to use - :class:`~python:collections.OrderedDict` (as shown above) to - ensure the possible status codes are displayed in the - correct order, when applicable. - - .. attribute:: status_code - - Status code for the batch as a whole. This indicates whether - the batch is "okay" and ready to execute, or (why) not etc. - - This must correspond to an existing key within the - :attr:`STATUS` dict. - - See also :attr:`status_text`. - - .. attribute:: status_text - - Text which may (briefly) further explain the batch - :attr:`status_code`, if needed. - - For example, assuming built-in default :attr:`STATUS` - definition:: - - batch.status_code = batch.STATUS_INCOMPLETE - batch.status_text = "cannot execute batch because it is missing something" - - .. attribute:: created - - When the batch was first created. - - .. attribute:: created_by - - Reference to the :class:`~wuttjamaican.db.model.auth.User` who - first created the batch. - - .. attribute:: executed - - When the batch was executed. - - .. attribute:: executed_by - - Reference to the :class:`~wuttjamaican.db.model.auth.User` who - executed the batch. - - """ - - @declared_attr - def __table_args__(cls): - return cls.__default_table_args__() - - @classmethod - def __default_table_args__(cls): - return cls.__batch_table_args__() - - @classmethod - def __batch_table_args__(cls): - return ( - sa.ForeignKeyConstraint(['created_by_uuid'], ['user.uuid']), - sa.ForeignKeyConstraint(['executed_by_uuid'], ['user.uuid']), - ) - - @declared_attr - def batch_type(cls): - return cls.__tablename__ - - uuid = uuid_column() - - id = sa.Column(sa.Integer(), nullable=False) - description = sa.Column(sa.String(length=255), nullable=True) - notes = sa.Column(sa.Text(), nullable=True) - row_count = sa.Column(sa.Integer(), nullable=True, default=0) - - STATUS_INCOMPLETE = 1 - STATUS_EXECUTABLE = 2 - - STATUS = { - STATUS_INCOMPLETE : "incomplete", - STATUS_EXECUTABLE : "executable", - } - - status_code = sa.Column(sa.Integer(), nullable=True) - status_text = sa.Column(sa.String(length=255), nullable=True) - - created = sa.Column(sa.DateTime(timezone=True), nullable=False, - default=datetime.datetime.now) - created_by_uuid = sa.Column(UUID(), nullable=False) - - @declared_attr - def created_by(cls): - return orm.relationship( - User, - primaryjoin=lambda: User.uuid == cls.created_by_uuid, - foreign_keys=lambda: [cls.created_by_uuid]) - - - executed = sa.Column(sa.DateTime(timezone=True), nullable=True) - executed_by_uuid = sa.Column(UUID(), nullable=True) - - @declared_attr - def executed_by(cls): - return orm.relationship( - User, - primaryjoin=lambda: User.uuid == cls.executed_by_uuid, - foreign_keys=lambda: [cls.executed_by_uuid]) - - def __repr__(self): - cls = self.__class__.__name__ - return f"{cls}(uuid={repr(self.uuid)})" - - def __str__(self): - return self.id_str if self.id else "(new)" - - @property - def id_str(self): - """ - Property which returns the :attr:`id` as a string, zero-padded - to 8 digits:: - - batch.id = 42 - print(batch.id_str) # => '00000042' - """ - if self.id: - return f'{self.id:08d}' - - -class BatchRowMixin: - """ - Mixin base class for :term:`data models ` which - represent a :term:`batch row`. - - See also :class:`BatchMixin` which should be used for the (parent) - batch model. - - .. attribute:: __batch_class__ - - Reference to the :term:`data model` for the parent - :term:`batch` class. - - This will be a subclass of :class:`BatchMixin` (among other - classes). - - When defining the batch row model, you must set this attribute - explicitly! And then :attr:`BatchMixin.__row_class__` will be - set automatically to match. - - .. attribute:: batch - - Reference to the parent :term:`batch` to which the row belongs. - - This will be an instance of :class:`BatchMixin` (among other - base classes). - - .. attribute:: sequence - - Sequence (aka. line) number for the row, within the parent - batch. This is 1-based so the first row has sequence 1, etc. - - .. attribute:: STATUS - - Dict of possible row status codes and their human-readable - names. - - Each key will be a possible :attr:`status_code` and the - corresponding value will be the human-readable name. - - See also :attr:`status_text` for when more detail/subtlety is - needed. - - Typically each "key" (code) is also defined as its own - "constant" on the model class. For instance:: - - from collections import OrderedDict - from wuttjamaican.db import model - - class MyBatchRow(model.BatchRowMixin, model.Base): - \""" my custom batch row \""" - - STATUS_INVALID = 1 - STATUS_GOOD_TO_GO = 2 - - STATUS = OrderedDict([ - (STATUS_INVALID, "invalid"), - (STATUS_GOOD_TO_GO, "good to go"), - ]) - - # TODO: column definitions... - - Whereas there is a built-in default for the - :attr:`BatchMixin.STATUS`, there is no built-in default defined - for the ``BatchRowMixin.STATUS``. Subclass must overwrite the - definition entirely, in similar fashion to above. - - .. note:: - There is not any built-in logic around these integer codes; - subclass can use any the developer prefers. - - Of course, once you define one, if any live batches use it, - you should not then change its fundamental meaning (although - you can change the human-readable text). - - It's recommended to use - :class:`~python:collections.OrderedDict` (as shown above) to - ensure the possible status codes are displayed in the - correct order, when applicable. - - .. attribute:: status_code - - Current status code for the row. This indicates if the row is - "good to go" or has "warnings" or is outright "invalid" etc. - - This must correspond to an existing key within the - :attr:`STATUS` dict. - - See also :attr:`status_text`. - - .. attribute:: status_text - - Text which may (briefly) further explain the row - :attr:`status_code`, if needed. - - For instance, assuming the example :attr:`STATUS` definition - shown above:: - - row.status_code = row.STATUS_INVALID - row.status_text = "input data for this row is missing fields: foo, bar" - - .. attribute:: modified - - Last modification time of the row. This should be - automatically set when the row is first created, as well as - anytime it's updated thereafter. - """ - - uuid = uuid_column() - - @declared_attr - def __table_args__(cls): - return cls.__default_table_args__() - - @classmethod - def __default_table_args__(cls): - return cls.__batchrow_table_args__() - - @classmethod - def __batchrow_table_args__(cls): - batch_table = cls.__batch_class__.__tablename__ - return ( - sa.ForeignKeyConstraint(['batch_uuid'], [f'{batch_table}.uuid']), - ) - - batch_uuid = sa.Column(UUID(), nullable=False) - - @declared_attr - def batch(cls): - batch_class = cls.__batch_class__ - row_class = cls - batch_class.__row_class__ = row_class - - # must establish `Batch.rows` here instead of from within the - # Batch above, because BatchRow class doesn't yet exist above. - batch_class.rows = orm.relationship( - row_class, - order_by=lambda: row_class.sequence, - collection_class=ordering_list('sequence', count_from=1), - cascade='all, delete-orphan', - back_populates='batch') - - # now, here's the `BatchRow.batch` - return orm.relationship( - batch_class, - back_populates='rows') - - sequence = sa.Column(sa.Integer(), nullable=False) - - STATUS = {} - - status_code = sa.Column(sa.Integer(), nullable=True) - status_text = sa.Column(sa.String(length=255), nullable=True) - - modified = sa.Column(sa.DateTime(timezone=True), nullable=True, - default=datetime.datetime.now, - onupdate=datetime.datetime.now) diff --git a/tests/db/model/test_batch.py b/tests/db/model/test_batch.py deleted file mode 100644 index fa16d25..0000000 --- a/tests/db/model/test_batch.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8; -*- - -import uuid as _uuid - -from wuttjamaican.testing import DataTestCase - -try: - import sqlalchemy as sa - from wuttjamaican.db import model - from wuttjamaican.db.model import batch as mod -except ImportError: - pass -else: - - class TestBatchMixin(DataTestCase): - - def test_basic(self): - - class MyBatch(mod.BatchMixin, model.Base): - __tablename__ = 'testing_mybatch' - - model.Base.metadata.create_all(bind=self.session.bind) - metadata = sa.MetaData() - metadata.reflect(self.session.bind) - self.assertIn('testing_mybatch', metadata.tables) - - batch = MyBatch(id=42, uuid=_uuid.UUID('0675cdac-ffc9-7690-8000-6023de1c8cfd')) - self.assertEqual(repr(batch), "MyBatch(uuid=UUID('0675cdac-ffc9-7690-8000-6023de1c8cfd'))") - self.assertEqual(str(batch), "00000042") - - - class TestBatchRowMixin(DataTestCase): - - def test_basic(self): - - class MyBatch2(mod.BatchMixin, model.Base): - __tablename__ = 'testing_mybatch2' - - class MyBatchRow2(mod.BatchRowMixin, model.Base): - __tablename__ = 'testing_mybatch_row2' - __batch_class__ = MyBatch2 - - model.Base.metadata.create_all(bind=self.session.bind) - metadata = sa.MetaData() - metadata.reflect(self.session.bind) - self.assertIn('testing_mybatch2', metadata.tables) - self.assertIn('testing_mybatch_row2', metadata.tables) - - # nb. this gives coverage but doesn't really test much - batch = MyBatch2(id=42, uuid=_uuid.UUID('0675cdac-ffc9-7690-8000-6023de1c8cfd')) - row = MyBatchRow2() - batch.rows.append(row) diff --git a/tests/db/test_handler.py b/tests/db/test_handler.py deleted file mode 100644 index e28e813..0000000 --- a/tests/db/test_handler.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8; -*- - -from unittest.mock import patch, MagicMock - -from wuttjamaican.testing import DataTestCase - -try: - import sqlalchemy as sa - from wuttjamaican.db import handler as mod -except ImportError: - pass -else: - - class TestDatabaseHandler(DataTestCase): - - def make_handler(self, **kwargs): - return mod.DatabaseHandler(self.config, **kwargs) - - def test_next_counter_value_sqlite(self): - handler = self.make_handler() - - # counter table should not exist yet - metadata = sa.MetaData() - metadata.reflect(self.session.bind) - self.assertNotIn('_counter_testing', metadata.tables) - - # using sqlite as backend, should make table for counter - value = handler.next_counter_value(self.session, 'testing') - self.assertEqual(value, 1) - - # counter table should exist now - metadata.reflect(self.session.bind) - self.assertIn('_counter_testing', metadata.tables) - - # counter increments okay - value = handler.next_counter_value(self.session, 'testing') - self.assertEqual(value, 2) - value = handler.next_counter_value(self.session, 'testing') - self.assertEqual(value, 3) - - def test_next_counter_value_postgres(self): - handler = self.make_handler() - - # counter table should not exist - metadata = sa.MetaData() - metadata.reflect(self.session.bind) - self.assertNotIn('_counter_testing', metadata.tables) - - # nb. we have to pretty much mock this out, can't really - # test true sequence behavior for postgres since tests are - # using sqlite backend. - - # using postgres as backend, should use "sequence" - with patch.object(self.session.bind.url, 'get_dialect') as get_dialect: - get_dialect.return_value.name = 'postgresql' - with patch.object(self.session, 'execute') as execute: - execute.return_value.scalar.return_value = 1 - value = handler.next_counter_value(self.session, 'testing') - self.assertEqual(value, 1) - execute.return_value.scalar.assert_called_once_with() - - # counter table should still not exist - metadata.reflect(self.session.bind) - self.assertNotIn('_counter_testing', metadata.tables) diff --git a/tests/test_app.py b/tests/test_app.py index 71a4066..be67509 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -434,14 +434,6 @@ app_title = WuttaTest dt = datetime.datetime(2024, 12, 11, 8, 30, tzinfo=datetime.timezone.utc) self.assertEqual(self.app.render_datetime(dt), '2024-12-11 08:30+0000') - def test_render_time_ago(self): - with patch.object(mod, 'humanize') as humanize: - humanize.naturaltime.return_value = 'now' - now = datetime.datetime.now() - result = self.app.render_time_ago(now) - self.assertEqual(result, 'now') - humanize.naturaltime.assert_called_once_with(now) - def test_get_person(self): people = self.app.get_people_handler() with patch.object(people, 'get_person') as get_person: @@ -456,15 +448,6 @@ app_title = WuttaTest auth = self.app.get_auth_handler() self.assertIsInstance(auth, AuthHandler) - def test_get_db_handler(self): - try: - from wuttjamaican.db.handler import DatabaseHandler - except ImportError: - pytest.skip("test not relevant without sqlalchemy") - - db = self.app.get_db_handler() - self.assertIsInstance(db, DatabaseHandler) - def test_get_email_handler(self): from wuttjamaican.email import EmailHandler diff --git a/tests/test_batch.py b/tests/test_batch.py deleted file mode 100644 index 6075fd0..0000000 --- a/tests/test_batch.py +++ /dev/null @@ -1,198 +0,0 @@ -# -*- coding: utf-8; -*- - -import os -from unittest.mock import patch - -from wuttjamaican import batch as mod - -try: - import sqlalchemy as sa - from wuttjamaican.db import model - from wuttjamaican.testing import DataTestCase -except ImportError: - pass -else: - - class MockBatch(model.BatchMixin, model.Base): - __tablename__ = 'testing_batch_mock' - - class MockBatchRow(model.BatchRowMixin, model.Base): - __tablename__ = 'testing_batch_mock_row' - __batch_class__ = MockBatch - - class MockBatchHandler(mod.BatchHandler): - model_class = MockBatch - - class TestBatchHandler(DataTestCase): - - def make_handler(self, **kwargs): - return MockBatchHandler(self.config, **kwargs) - - def test_model_class(self): - handler = mod.BatchHandler(self.config) - self.assertRaises(NotImplementedError, getattr, handler, 'model_class') - - def test_batch_type(self): - with patch.object(mod.BatchHandler, 'model_class', new=MockBatch): - handler = mod.BatchHandler(self.config) - self.assertEqual(handler.batch_type, 'testing_batch_mock') - - def test_make_batch(self): - handler = self.make_handler() - batch = handler.make_batch(self.session) - self.assertIsInstance(batch, MockBatch) - - def test_consume_batch_id(self): - handler = self.make_handler() - - first = handler.consume_batch_id(self.session) - second = handler.consume_batch_id(self.session) - self.assertEqual(second, first + 1) - - third = handler.consume_batch_id(self.session, as_str=True) - self.assertEqual(third, f'{first + 2:08d}') - - def test_get_data_path(self): - model = self.app.model - user = model.User(username='barney') - self.session.add(user) - - with patch.object(mod.BatchHandler, 'model_class', new=MockBatch): - handler = self.make_handler() - - # root storage (default) - with patch.object(self.app, 'get_appdir', return_value=self.tempdir): - path = handler.get_data_path() - self.assertEqual(path, os.path.join(self.tempdir, 'data', 'batch', 'testing_batch_mock')) - - # root storage (configured) - self.config.setdefault('wutta.batch.storage_path', self.tempdir) - path = handler.get_data_path() - self.assertEqual(path, os.path.join(self.tempdir, 'testing_batch_mock')) - - batch = handler.make_batch(self.session, created_by=user) - self.session.add(batch) - self.session.flush() - - # batch-specific - path = handler.get_data_path(batch) - uuid = batch.uuid.hex - final = os.path.join(uuid[-2:], uuid[:-2]) - self.assertEqual(path, os.path.join(self.tempdir, 'testing_batch_mock', final)) - - # with filename - path = handler.get_data_path(batch, 'input.csv') - self.assertEqual(path, os.path.join(self.tempdir, 'testing_batch_mock', final, 'input.csv')) - - # makedirs - path = handler.get_data_path(batch) - self.assertFalse(os.path.exists(path)) - path = handler.get_data_path(batch, makedirs=True) - self.assertTrue(os.path.exists(path)) - - def test_should_populate(self): - handler = self.make_handler() - batch = handler.make_batch(self.session) - self.assertFalse(handler.should_populate(batch)) - - def test_do_populate(self): - handler = self.make_handler() - batch = handler.make_batch(self.session) - # nb. coverage only; tests nothing - handler.do_populate(batch) - - def test_make_row(self): - handler = self.make_handler() - row = handler.make_row() - self.assertIsInstance(row, MockBatchRow) - - def test_add_row(self): - handler = self.make_handler() - batch = handler.make_batch(self.session) - self.session.add(batch) - row = handler.make_row() - self.assertIsNone(batch.row_count) - handler.add_row(batch, row) - self.assertEqual(batch.row_count, 1) - - def test_do_execute(self): - model = self.app.model - user = model.User(username='barney') - self.session.add(user) - - handler = self.make_handler() - batch = handler.make_batch(self.session, created_by=user) - self.session.add(batch) - self.session.flush() - - # error if execution not allowed - with patch.object(handler, 'why_not_execute', return_value="bad batch"): - self.assertRaises(RuntimeError, handler.do_execute, batch, user) - - # nb. coverage only; tests nothing - self.assertIsNone(batch.executed) - self.assertIsNone(batch.executed_by) - handler.do_execute(batch, user) - self.assertIsNotNone(batch.executed) - self.assertIs(batch.executed_by, user) - - # error if execution already happened - self.assertRaises(ValueError, handler.do_execute, batch, user) - - def test_do_delete(self): - model = self.app.model - handler = self.make_handler() - - user = model.User(username='barney') - self.session.add(user) - - # simple delete - batch = handler.make_batch(self.session, created_by=user) - self.session.add(batch) - self.session.flush() - self.assertEqual(self.session.query(MockBatch).count(), 1) - handler.do_delete(batch, user) - self.assertEqual(self.session.query(MockBatch).count(), 0) - - # delete w/ rows - batch = handler.make_batch(self.session, created_by=user) - self.session.add(batch) - for i in range(5): - row = handler.make_row() - handler.add_row(batch, row) - self.session.flush() - self.assertEqual(self.session.query(MockBatch).count(), 1) - handler.do_delete(batch, user) - self.assertEqual(self.session.query(MockBatch).count(), 0) - - # delete w/ files - self.config.setdefault('wutta.batch.storage_path', self.tempdir) - batch = handler.make_batch(self.session, created_by=user) - self.session.add(batch) - self.session.flush() - path = handler.get_data_path(batch, 'data.txt', makedirs=True) - with open(path, 'wt') as f: - f.write('foo=bar') - self.assertEqual(self.session.query(MockBatch).count(), 1) - path = handler.get_data_path(batch) - self.assertTrue(os.path.exists(path)) - handler.do_delete(batch, user) - self.assertEqual(self.session.query(MockBatch).count(), 0) - self.assertFalse(os.path.exists(path)) - - # delete w/ files (dry-run) - self.config.setdefault('wutta.batch.storage_path', self.tempdir) - batch = handler.make_batch(self.session, created_by=user) - self.session.add(batch) - self.session.flush() - path = handler.get_data_path(batch, 'data.txt', makedirs=True) - with open(path, 'wt') as f: - f.write('foo=bar') - self.assertEqual(self.session.query(MockBatch).count(), 1) - path = handler.get_data_path(batch) - self.assertTrue(os.path.exists(path)) - handler.do_delete(batch, user, dry_run=True) - # nb. batch appears missing from session even in dry-run - self.assertEqual(self.session.query(MockBatch).count(), 0) - # nb. but its files remain intact - self.assertTrue(os.path.exists(path))