From 51accc5a93af91978c84345023f392edd50be104 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 13 Dec 2024 17:26:07 -0600 Subject: [PATCH 1/4] feat: add basic db handler, for tracking counter values more to come i'm sure, this is all i need so far --- docs/api/wuttjamaican.db.handler.rst | 6 +++ docs/glossary.rst | 8 +++ docs/index.rst | 1 + src/wuttjamaican/app.py | 18 ++++++- src/wuttjamaican/db/handler.py | 73 ++++++++++++++++++++++++++++ tests/db/test_handler.py | 64 ++++++++++++++++++++++++ tests/test_app.py | 9 ++++ 7 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 docs/api/wuttjamaican.db.handler.rst create mode 100644 src/wuttjamaican/db/handler.py create mode 100644 tests/db/test_handler.py diff --git a/docs/api/wuttjamaican.db.handler.rst b/docs/api/wuttjamaican.db.handler.rst new file mode 100644 index 0000000..6ae56d3 --- /dev/null +++ b/docs/api/wuttjamaican.db.handler.rst @@ -0,0 +1,6 @@ + +``wuttjamaican.db.handler`` +=========================== + +.. automodule:: wuttjamaican.db.handler + :members: diff --git a/docs/glossary.rst b/docs/glossary.rst index 777ba0b..b6cb19f 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -128,6 +128,14 @@ Glossary Most :term:`apps` will have at least one :term:`app database`. See also :doc:`narr/db/index`. + db handler + The :term:`handler` responsible for various operations involving + the :term:`app database` (and possibly other :term:`databases + `). + + See also the :class:`~wuttjamaican.db.handler.DatabaseHandler` + base class. + db session The "session" is a SQLAlchemy abstraction for an open database connection, essentially. diff --git a/docs/index.rst b/docs/index.rst index cd2064f..c62b34a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -71,6 +71,7 @@ Contents api/wuttjamaican.conf api/wuttjamaican.db api/wuttjamaican.db.conf + api/wuttjamaican.db.handler api/wuttjamaican.db.model api/wuttjamaican.db.model.auth api/wuttjamaican.db.model.base diff --git a/src/wuttjamaican/app.py b/src/wuttjamaican/app.py index 7a7000c..19766eb 100644 --- a/src/wuttjamaican/app.py +++ b/src/wuttjamaican/app.py @@ -81,6 +81,7 @@ class AppHandler: default_model_spec = 'wuttjamaican.db.model' default_enum_spec = 'wuttjamaican.enum' default_auth_handler_spec = 'wuttjamaican.auth:AuthHandler' + default_db_handler_spec = 'wuttjamaican.db.handler:DatabaseHandler' default_email_handler_spec = 'wuttjamaican.email:EmailHandler' default_install_handler_spec = 'wuttjamaican.install:InstallHandler' default_people_handler_spec = 'wuttjamaican.people:PeopleHandler' @@ -493,7 +494,7 @@ class AppHandler: def make_true_uuid(self): """ - Generate a new v7 UUID value. + Generate a new UUID value. By default this simply calls :func:`wuttjamaican.util.make_true_uuid()`. @@ -514,7 +515,7 @@ class AppHandler: def make_uuid(self): """ - Generate a new v7 UUID value. + Generate a new UUID value. By default this simply calls :func:`wuttjamaican.util.make_uuid()`. @@ -730,6 +731,19 @@ class AppHandler: self.handlers['auth'] = factory(self.config, **kwargs) return self.handlers['auth'] + def get_db_handler(self, **kwargs): + """ + Get the configured :term:`db handler`. + + :rtype: :class:`~wuttjamaican.db.handler.DatabaseHandler` + """ + if 'db' not in self.handlers: + spec = self.config.get(f'{self.appname}.db.handler', + default=self.default_db_handler_spec) + factory = self.load_object(spec) + self.handlers['db'] = factory(self.config, **kwargs) + return self.handlers['db'] + def get_email_handler(self, **kwargs): """ Get the configured :term:`email handler`. diff --git a/src/wuttjamaican/db/handler.py b/src/wuttjamaican/db/handler.py new file mode 100644 index 0000000..7c745d8 --- /dev/null +++ b/src/wuttjamaican/db/handler.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttJamaican -- Base package for Wutta Framework +# Copyright © 2024 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Database Handler +""" + +import sqlalchemy as sa + +from wuttjamaican.app import GenericHandler + + +class DatabaseHandler(GenericHandler): + """ + Base class and default implementation for the :term:`db handler`. + """ + + def next_counter_value(self, session, key): + """ + Return the next counter value for the given key. + + If the DB backend is PostgreSQL, then a proper "sequence" is + used for the counter. + + All other backends use a "fake" sequence by creating a + dedicated table with auto-increment primary key, to provide + the counter. + + :param session: Current :term:`db session`. + + :param key: Unique key indicating the counter for which the + next value should be fetched. + + :returns: Next value as integer. + """ + dialect = session.bind.url.get_dialect().name + + # postgres uses "true" native sequence + if dialect == 'postgresql': + sql = f"create sequence if not exists {key}_seq" + session.execute(sa.text(sql)) + sql = f"select nextval('{key}_seq')" + value = session.execute(sa.text(sql)).scalar() + return value + + # otherwise use "magic" workaround + engine = session.bind + metadata = sa.MetaData() + table = sa.Table(f'_counter_{key}', metadata, + sa.Column('value', sa.Integer(), primary_key=True)) + table.create(engine, checkfirst=True) + with engine.begin() as cxn: + result = cxn.execute(table.insert()) + return result.lastrowid diff --git a/tests/db/test_handler.py b/tests/db/test_handler.py new file mode 100644 index 0000000..e28e813 --- /dev/null +++ b/tests/db/test_handler.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8; -*- + +from unittest.mock import patch, MagicMock + +from wuttjamaican.testing import DataTestCase + +try: + import sqlalchemy as sa + from wuttjamaican.db import handler as mod +except ImportError: + pass +else: + + class TestDatabaseHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.DatabaseHandler(self.config, **kwargs) + + def test_next_counter_value_sqlite(self): + handler = self.make_handler() + + # counter table should not exist yet + metadata = sa.MetaData() + metadata.reflect(self.session.bind) + self.assertNotIn('_counter_testing', metadata.tables) + + # using sqlite as backend, should make table for counter + value = handler.next_counter_value(self.session, 'testing') + self.assertEqual(value, 1) + + # counter table should exist now + metadata.reflect(self.session.bind) + self.assertIn('_counter_testing', metadata.tables) + + # counter increments okay + value = handler.next_counter_value(self.session, 'testing') + self.assertEqual(value, 2) + value = handler.next_counter_value(self.session, 'testing') + self.assertEqual(value, 3) + + def test_next_counter_value_postgres(self): + handler = self.make_handler() + + # counter table should not exist + metadata = sa.MetaData() + metadata.reflect(self.session.bind) + self.assertNotIn('_counter_testing', metadata.tables) + + # nb. we have to pretty much mock this out, can't really + # test true sequence behavior for postgres since tests are + # using sqlite backend. + + # using postgres as backend, should use "sequence" + with patch.object(self.session.bind.url, 'get_dialect') as get_dialect: + get_dialect.return_value.name = 'postgresql' + with patch.object(self.session, 'execute') as execute: + execute.return_value.scalar.return_value = 1 + value = handler.next_counter_value(self.session, 'testing') + self.assertEqual(value, 1) + execute.return_value.scalar.assert_called_once_with() + + # counter table should still not exist + metadata.reflect(self.session.bind) + self.assertNotIn('_counter_testing', metadata.tables) diff --git a/tests/test_app.py b/tests/test_app.py index be67509..ca8c952 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -448,6 +448,15 @@ app_title = WuttaTest auth = self.app.get_auth_handler() self.assertIsInstance(auth, AuthHandler) + def test_get_db_handler(self): + try: + from wuttjamaican.db.handler import DatabaseHandler + except ImportError: + pytest.skip("test not relevant without sqlalchemy") + + db = self.app.get_db_handler() + self.assertIsInstance(db, DatabaseHandler) + def test_get_email_handler(self): from wuttjamaican.email import EmailHandler From a514d9cfbaa4d0a2bc4e491b3eb564f700a64809 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 13 Dec 2024 20:38:00 -0600 Subject: [PATCH 2/4] feat: add basic batch feature, data model and partial handler hopefully data model is complete enough for now, but handler does not yet have all methods, e.g. execute() --- docs/api/wuttjamaican.batch.rst | 6 + docs/api/wuttjamaican.db.model.batch.rst | 6 + docs/glossary.rst | 36 ++ docs/index.rst | 2 + src/wuttjamaican/batch.py | 217 ++++++++++++ src/wuttjamaican/db/model/__init__.py | 9 +- src/wuttjamaican/db/model/batch.py | 416 +++++++++++++++++++++++ tests/db/model/test_batch.py | 52 +++ tests/test_batch.py | 70 ++++ 9 files changed, 813 insertions(+), 1 deletion(-) create mode 100644 docs/api/wuttjamaican.batch.rst create mode 100644 docs/api/wuttjamaican.db.model.batch.rst create mode 100644 src/wuttjamaican/batch.py create mode 100644 src/wuttjamaican/db/model/batch.py create mode 100644 tests/db/model/test_batch.py create mode 100644 tests/test_batch.py diff --git a/docs/api/wuttjamaican.batch.rst b/docs/api/wuttjamaican.batch.rst new file mode 100644 index 0000000..a51fd77 --- /dev/null +++ b/docs/api/wuttjamaican.batch.rst @@ -0,0 +1,6 @@ + +``wuttjamaican.batch`` +====================== + +.. automodule:: wuttjamaican.batch + :members: diff --git a/docs/api/wuttjamaican.db.model.batch.rst b/docs/api/wuttjamaican.db.model.batch.rst new file mode 100644 index 0000000..050c04d --- /dev/null +++ b/docs/api/wuttjamaican.db.model.batch.rst @@ -0,0 +1,6 @@ + +``wuttjamaican.db.model.batch`` +=============================== + +.. automodule:: wuttjamaican.db.model.batch + :members: diff --git a/docs/glossary.rst b/docs/glossary.rst index b6cb19f..137aefe 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -76,6 +76,42 @@ Glossary See also :class:`~wuttjamaican.auth.AuthHandler`. + batch + This refers to a process whereby bulk data operations may be + performed, with preview and other tools to allow the user to + refine as needed before "executing" the batch. + + The term "batch" may refer to such a feature overall, or the + :term:`data model` used, or the specific data for a single batch, + etc. + + See also :term:`batch handler` and :term:`batch row`, and the + :class:`~wuttjamaican.db.model.batch.BatchMixin` base class. + + batch handler + This refers to a :term:`handler` meant to process a given type of + :term:`batch`. + + There may be multiple handlers registered for a given + :term:`batch type`, but (usually) only one will be configured for + use. + + batch row + A row of data within a :term:`batch`. + + May also refer to the :term:`data model` class used for such a row. + + See also the :class:`~wuttjamaican.db.model.batch.BatchRowMixin` + base class. + + batch type + This term is used to distinguish :term:`batches ` according + to which underlying table is used to store their data, essentially. + + For instance a "pricing batch" would use one table, whereas an + "inventory batch" would use another. And each "type" would be + managed by its own :term:`batch handler`. + command A top-level command line interface for the app. Note that top-level commands don't usually "do" anything per se, and are diff --git a/docs/index.rst b/docs/index.rst index c62b34a..896a543 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Contents api/wuttjamaican.app api/wuttjamaican.auth + api/wuttjamaican.batch api/wuttjamaican.cli api/wuttjamaican.cli.base api/wuttjamaican.cli.make_appdir @@ -75,6 +76,7 @@ Contents api/wuttjamaican.db.model api/wuttjamaican.db.model.auth api/wuttjamaican.db.model.base + api/wuttjamaican.db.model.batch api/wuttjamaican.db.model.upgrades api/wuttjamaican.db.sess api/wuttjamaican.db.util diff --git a/src/wuttjamaican/batch.py b/src/wuttjamaican/batch.py new file mode 100644 index 0000000..ec0f708 --- /dev/null +++ b/src/wuttjamaican/batch.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttJamaican -- Base package for Wutta Framework +# Copyright © 2023-2024 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Batch Handlers +""" + +from wuttjamaican.app import GenericHandler + + +class BatchHandler(GenericHandler): + """ + Base class and *partial* default implementation for :term:`batch + handlers `. + + This handler class "works as-is" but does not actually do + anything. Subclass must implement logic for various things as + needed, e.g.: + + * :attr:`model_class` + * :meth:`init_batch()` + * :meth:`should_populate()` + * :meth:`populate()` + * :meth:`refresh_row()` + """ + + @property + def model_class(self): + """ + Reference to the batch :term:`data model` class which this + batch handler is meant to work with. + + This is expected to be a subclass of + :class:`~wuttjamaican.db.model.batch.BatchMixin` (among other + classes). + + Subclass must define this; default is not implemented. + """ + raise NotImplementedError("You must set the 'model_class' attribute " + f"for class '{self.__class__.__name__}'") + + def make_batch(self, session, progress=None, **kwargs): + """ + Make and return a new batch (:attr:`model_class`) instance. + + This will create the new batch, and auto-assign its + :attr:`~wuttjamaican.db.model.batch.BatchMixin.id` value + (unless caller specifies it) by calling + :meth:`consume_batch_id()`. + + It then will call :meth:`init_batch()` to perform any custom + initialization needed. + + Therefore callers should use this ``make_batch()`` method, but + subclass should override :meth:`init_batch()` instead (if + needed). + + :param session: Current :term:`db session`. + + :param progress: Optional progress indicator factory. + + :param \**kwargs: Additional kwargs to pass to the batch + constructor. + + :returns: New batch; instance of :attr:`model_class`. + """ + # generate new ID unless caller specifies + if 'id' not in kwargs: + kwargs['id'] = self.consume_batch_id(session) + + # make batch + batch = self.model_class(**kwargs) + self.init_batch(batch, session=session, progress=progress, **kwargs) + return batch + + def consume_batch_id(self, session, as_str=False): + """ + Fetch a new batch ID from the counter, and return it. + + This may be called automatically from :meth:`make_batch()`. + + :param session: Current :term:`db session`. + + :param as_str: Indicates the return value should be a string + instead of integer. + + :returns: Batch ID as integer, or zero-padded 8-char string. + """ + db = self.app.get_db_handler() + batch_id = db.next_counter_value(session, 'batch_id') + if as_str: + return f'{batch_id:08d}' + return batch_id + + def init_batch(self, batch, session=None, progress=None, **kwargs): + """ + Initialize a new batch. + + This is called automatically from :meth:`make_batch()`. + + Default logic does nothing; subclass should override if needed. + + .. note:: + *Population* of the new batch should **not** happen here; + see instead :meth:`populate()`. + """ + + def should_populate(self, batch): + """ + Must return true or false, indicating whether the given batch + should be populated from initial data source(s). + + So, true means fill the batch with data up front - by calling + :meth:`do_populate()` - and false means the batch will start + empty. + + Default logic here always return false; subclass should + override if needed. + """ + return False + + def do_populate(self, batch, progress=None): + """ + Populate the batch from initial data source(s). + + This method is a convenience wrapper, which ultimately will + call :meth:`populate()` for the implementation logic. + + Therefore callers should use this ``do_populate()`` method, + but subclass should override :meth:`populate()` instead (if + needed). + + See also :meth:`should_populate()` - you should check that + before calling ``do_populate()``. + """ + self.populate(batch, progress=progress) + + def populate(self, batch, progress=None): + """ + Populate the batch from initial data source(s). + + It is assumed that the data source(s) to be used will be known + by inspecting various properties of the batch itself. + + Subclass should override this method to provide the + implementation logic. It may populate some batches + differently based on the batch attributes, or it may populate + them all the same. Whatever is needed. + + Callers should always use :meth:`do_populate()` instead of + calling ``populate()`` directly. + """ + + def make_row(self, **kwargs): + """ + Make a new row for the batch. This will be an instance of + :attr:`~wuttjamaican.db.model.batch.BatchMixin.__row_class__`. + + Note that the row will **not** be added to the batch; that + should be done with :meth:`add_row()`. + + :returns: A new row object, which does *not* yet belong to any batch. + """ + return self.model_class.__row_class__(**kwargs) + + def add_row(self, batch, row): + """ + Add the given row to the given batch. + + This assumes a *new* row which does not yet belong to a batch, + as returned by :meth:`make_row()`. + + It will add it to batch + :attr:`~wuttjamaican.db.model.batch.BatchMixin.rows`, call + :meth:`refresh_row()` for it, and update the + :attr:`~wuttjamaican.db.model.batch.BatchMixin.row_count`. + """ + session = self.app.get_session(batch) + with session.no_autoflush: + batch.rows.append(row) + self.refresh_row(row) + batch.row_count = (batch.row_count or 0) + 1 + + def refresh_row(self, row): + """ + Update the given batch row as needed, to reflect latest data. + + This method is a bit of a catch-all in that it could be used + to do any of the following (etc.): + + * fetch latest "live" data for comparison with batch input data + * calculate some data values based on the previous step + * set row status based on other row attributes + + This method is called when the row is first added to the batch + via :meth:`add_row()` - but may be called multiple times after + that depending on the workflow. + """ diff --git a/src/wuttjamaican/db/model/__init__.py b/src/wuttjamaican/db/model/__init__.py index b0c5255..6a9883b 100644 --- a/src/wuttjamaican/db/model/__init__.py +++ b/src/wuttjamaican/db/model/__init__.py @@ -30,6 +30,7 @@ This namespace exposes the following: * :class:`~wuttjamaican.db.model.base.Base` * :func:`~wuttjamaican.db.util.uuid_column()` * :func:`~wuttjamaican.db.util.uuid_fk_column()` +* :class:`~wuttjamaican.db.util.UUID` And the :term:`data models `: @@ -40,10 +41,16 @@ And the :term:`data models `: * :class:`~wuttjamaican.db.model.auth.User` * :class:`~wuttjamaican.db.model.auth.UserRole` * :class:`~wuttjamaican.db.model.upgrades.Upgrade` + +And the :term:`batch` model base/mixin classes: + +* :class:`~wuttjamaican.db.model.batch.BatchMixin` +* :class:`~wuttjamaican.db.model.batch.BatchRowMixin` """ -from wuttjamaican.db.util import uuid_column, uuid_fk_column +from wuttjamaican.db.util import uuid_column, uuid_fk_column, UUID from .base import Base, Setting, Person from .auth import Role, Permission, User, UserRole from .upgrades import Upgrade +from .batch import BatchMixin, BatchRowMixin diff --git a/src/wuttjamaican/db/model/batch.py b/src/wuttjamaican/db/model/batch.py new file mode 100644 index 0000000..76801b8 --- /dev/null +++ b/src/wuttjamaican/db/model/batch.py @@ -0,0 +1,416 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttJamaican -- Base package for Wutta Framework +# Copyright © 2023-2024 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Batch data models +""" + +import datetime + +import sqlalchemy as sa +from sqlalchemy import orm +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.ext.orderinglist import ordering_list + +from wuttjamaican.db.model import uuid_column, uuid_fk_column, User +from wuttjamaican.db.util import UUID + + +class BatchMixin: + """ + Mixin base class for :term:`data models ` which + represent a :term:`batch`. + + See also :class:`BatchRowMixin` which should be used for the row + model. + + For a batch model (table) to be useful, at least one :term:`batch + handler` must be defined, which is able to process data for that + :term:`batch type`. + + .. attribute:: __row_class__ + + Reference to the specific :term:`data model` class used for the + :term:`batch rows `. + + This will be a subclass of :class:`BatchRowMixin` (among other + classes). + + When defining the batch model, you do not have to set this as + it will be assigned automatically based on + :attr:`BatchRowMixin.__batch_class__`. + + .. attribute:: id + + Numeric ID for the batch, unique across all batches (regardless + of type). + + See also :attr:`id_str`. + + .. attribute:: description + + Simple description for the batch. + + .. attribute:: notes + + Arbitrary notes for the batch. + + .. attribute:: rows + + List of data rows for the batch, aka. :term:`batch rows `. + + Each will be an instance of :class:`BatchRowMixin` (among other + base classes). + + .. attribute:: row_count + + Cached row count for the batch, i.e. how many :attr:`rows` it has. + + No guarantees perhaps, but this should ideally be accurate (it + ultimately depends on the :term:`batch handler` + implementation). + + .. attribute:: STATUS + + Dict of possible batch status codes and their human-readable + names. + + Each key will be a possible :attr:`status_code` and the + corresponding value will be the human-readable name. + + See also :attr:`status_text` for when more detail/subtlety is + needed. + + Typically each "key" (code) is also defined as its own + "constant" on the model class. For instance:: + + from collections import OrderedDict + from wuttjamaican.db import model + + class MyBatch(model.BatchMixin, model.Base): + \""" my custom batch \""" + + STATUS_INCOMPLETE = 1 + STATUS_EXECUTABLE = 2 + + STATUS = OrderedDict([ + (STATUS_INCOMPLETE, "incomplete"), + (STATUS_EXECUTABLE, "executable"), + ]) + + # TODO: column definitions... + + And in fact, the above status definition is the built-in + default. However it is expected for subclass to overwrite the + definition entirely (in similar fashion to above) when needed. + + .. note:: + There is not any built-in logic around these integer codes; + subclass can use any the developer prefers. + + Of course, once you define one, if any live batches use it, + you should not then change its fundamental meaning (although + you can change the human-readable text). + + It's recommended to use + :class:`~python:collections.OrderedDict` (as shown above) to + ensure the possible status codes are displayed in the + correct order, when applicable. + + .. attribute:: status_code + + Status code for the batch as a whole. This indicates whether + the batch is "okay" and ready to execute, or (why) not etc. + + This must correspond to an existing key within the + :attr:`STATUS` dict. + + See also :attr:`status_text`. + + .. attribute:: status_text + + Text which may (briefly) further explain the batch + :attr:`status_code`, if needed. + + For example, assuming built-in default :attr:`STATUS` + definition:: + + batch.status_code = batch.STATUS_INCOMPLETE + batch.status_text = "cannot execute batch because it is missing something" + + .. attribute:: created + + When the batch was first created. + + .. attribute:: created_by + + Reference to the :class:`~wuttjamaican.db.model.auth.User` who + first created the batch. + + .. attribute:: executed + + When the batch was executed. + + .. attribute:: executed_by + + Reference to the :class:`~wuttjamaican.db.model.auth.User` who + executed the batch. + + """ + + @declared_attr + def __table_args__(cls): + return cls.__default_table_args__() + + @classmethod + def __default_table_args__(cls): + return cls.__batch_table_args__() + + @classmethod + def __batch_table_args__(cls): + return ( + sa.ForeignKeyConstraint(['created_by_uuid'], ['user.uuid']), + sa.ForeignKeyConstraint(['executed_by_uuid'], ['user.uuid']), + ) + + @declared_attr + def batch_key(cls): + return cls.__tablename__ + + uuid = uuid_column() + + id = sa.Column(sa.Integer(), nullable=False) + description = sa.Column(sa.String(length=255), nullable=True) + notes = sa.Column(sa.Text(), nullable=True) + row_count = sa.Column(sa.Integer(), nullable=True, default=0) + + STATUS_INCOMPLETE = 1 + STATUS_EXECUTABLE = 2 + + STATUS = { + STATUS_INCOMPLETE : "incomplete", + STATUS_EXECUTABLE : "executable", + } + + status_code = sa.Column(sa.Integer(), nullable=True) + status_text = sa.Column(sa.String(length=255), nullable=True) + + created = sa.Column(sa.DateTime(timezone=True), nullable=False, + default=datetime.datetime.now) + created_by_uuid = sa.Column(UUID(), nullable=False) + + @declared_attr + def created_by(cls): + return orm.relationship( + User, + primaryjoin=lambda: User.uuid == cls.created_by_uuid, + foreign_keys=lambda: [cls.created_by_uuid]) + + + executed = sa.Column(sa.DateTime(timezone=True), nullable=True) + executed_by_uuid = sa.Column(UUID(), nullable=True) + + @declared_attr + def executed_by(cls): + return orm.relationship( + User, + primaryjoin=lambda: User.uuid == cls.executed_by_uuid, + foreign_keys=lambda: [cls.executed_by_uuid]) + + def __repr__(self): + cls = self.__class__.__name__ + return f"{cls}(uuid={repr(self.uuid)})" + + def __str__(self): + return self.id_str if self.id else "(new)" + + @property + def id_str(self): + """ + Property which returns the :attr:`id` as a string, zero-padded + to 8 digits:: + + batch.id = 42 + print(batch.id_str) # => '00000042' + """ + if self.id: + return f'{self.id:08d}' + + +class BatchRowMixin: + """ + Mixin base class for :term:`data models ` which + represent a :term:`batch row`. + + See also :class:`BatchMixin` which should be used for the (parent) + batch model. + + .. attribute:: __batch_class__ + + Reference to the :term:`data model` for the parent + :term:`batch` class. + + This will be a subclass of :class:`BatchMixin` (among other + classes). + + When defining the batch row model, you must set this attribute + explicitly! And then :attr:`BatchMixin.__row_class__` will be + set automatically to match. + + .. attribute:: batch + + Reference to the parent :term:`batch` to which the row belongs. + + This will be an instance of :class:`BatchMixin` (among other + base classes). + + .. attribute:: sequence + + Sequence (aka. line) number for the row, within the parent + batch. This is 1-based so the first row has sequence 1, etc. + + .. attribute:: STATUS + + Dict of possible row status codes and their human-readable + names. + + Each key will be a possible :attr:`status_code` and the + corresponding value will be the human-readable name. + + See also :attr:`status_text` for when more detail/subtlety is + needed. + + Typically each "key" (code) is also defined as its own + "constant" on the model class. For instance:: + + from collections import OrderedDict + from wuttjamaican.db import model + + class MyBatchRow(model.BatchRowMixin, model.Base): + \""" my custom batch row \""" + + STATUS_INVALID = 1 + STATUS_GOOD_TO_GO = 2 + + STATUS = OrderedDict([ + (STATUS_INVALID, "invalid"), + (STATUS_GOOD_TO_GO, "good to go"), + ]) + + # TODO: column definitions... + + Whereas there is a built-in default for the + :attr:`BatchMixin.STATUS`, there is no built-in default defined + for the ``BatchRowMixin.STATUS``. Subclass must overwrite the + definition entirely, in similar fashion to above. + + .. note:: + There is not any built-in logic around these integer codes; + subclass can use any the developer prefers. + + Of course, once you define one, if any live batches use it, + you should not then change its fundamental meaning (although + you can change the human-readable text). + + It's recommended to use + :class:`~python:collections.OrderedDict` (as shown above) to + ensure the possible status codes are displayed in the + correct order, when applicable. + + .. attribute:: status_code + + Current status code for the row. This indicates if the row is + "good to go" or has "warnings" or is outright "invalid" etc. + + This must correspond to an existing key within the + :attr:`STATUS` dict. + + See also :attr:`status_text`. + + .. attribute:: status_text + + Text which may (briefly) further explain the row + :attr:`status_code`, if needed. + + For instance, assuming the example :attr:`STATUS` definition + shown above:: + + row.status_code = row.STATUS_INVALID + row.status_text = "input data for this row is missing fields: foo, bar" + + .. attribute:: modified + + Last modification time of the row. This should be + automatically set when the row is first created, as well as + anytime it's updated thereafter. + """ + + uuid = uuid_column() + + @declared_attr + def __table_args__(cls): + return cls.__default_table_args__() + + @classmethod + def __default_table_args__(cls): + return cls.__batchrow_table_args__() + + @classmethod + def __batchrow_table_args__(cls): + batch_table = cls.__batch_class__.__tablename__ + return ( + sa.ForeignKeyConstraint(['batch_uuid'], [f'{batch_table}.uuid']), + ) + + batch_uuid = sa.Column(UUID(), nullable=False) + + @declared_attr + def batch(cls): + batch_class = cls.__batch_class__ + row_class = cls + batch_class.__row_class__ = row_class + + # must establish `Batch.rows` here instead of from within the + # Batch above, because BatchRow class doesn't yet exist above. + batch_class.rows = orm.relationship( + row_class, + order_by=lambda: row_class.sequence, + collection_class=ordering_list('sequence', count_from=1), + cascade='all, delete-orphan', + back_populates='batch') + + # now, here's the `BatchRow.batch` + return orm.relationship( + batch_class, + back_populates='rows') + + sequence = sa.Column(sa.Integer(), nullable=False) + + STATUS = {} + + status_code = sa.Column(sa.Integer(), nullable=True) + status_text = sa.Column(sa.String(length=255), nullable=True) + + modified = sa.Column(sa.DateTime(timezone=True), nullable=True, + default=datetime.datetime.now, + onupdate=datetime.datetime.now) diff --git a/tests/db/model/test_batch.py b/tests/db/model/test_batch.py new file mode 100644 index 0000000..fa16d25 --- /dev/null +++ b/tests/db/model/test_batch.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8; -*- + +import uuid as _uuid + +from wuttjamaican.testing import DataTestCase + +try: + import sqlalchemy as sa + from wuttjamaican.db import model + from wuttjamaican.db.model import batch as mod +except ImportError: + pass +else: + + class TestBatchMixin(DataTestCase): + + def test_basic(self): + + class MyBatch(mod.BatchMixin, model.Base): + __tablename__ = 'testing_mybatch' + + model.Base.metadata.create_all(bind=self.session.bind) + metadata = sa.MetaData() + metadata.reflect(self.session.bind) + self.assertIn('testing_mybatch', metadata.tables) + + batch = MyBatch(id=42, uuid=_uuid.UUID('0675cdac-ffc9-7690-8000-6023de1c8cfd')) + self.assertEqual(repr(batch), "MyBatch(uuid=UUID('0675cdac-ffc9-7690-8000-6023de1c8cfd'))") + self.assertEqual(str(batch), "00000042") + + + class TestBatchRowMixin(DataTestCase): + + def test_basic(self): + + class MyBatch2(mod.BatchMixin, model.Base): + __tablename__ = 'testing_mybatch2' + + class MyBatchRow2(mod.BatchRowMixin, model.Base): + __tablename__ = 'testing_mybatch_row2' + __batch_class__ = MyBatch2 + + model.Base.metadata.create_all(bind=self.session.bind) + metadata = sa.MetaData() + metadata.reflect(self.session.bind) + self.assertIn('testing_mybatch2', metadata.tables) + self.assertIn('testing_mybatch_row2', metadata.tables) + + # nb. this gives coverage but doesn't really test much + batch = MyBatch2(id=42, uuid=_uuid.UUID('0675cdac-ffc9-7690-8000-6023de1c8cfd')) + row = MyBatchRow2() + batch.rows.append(row) diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 0000000..b29c42b --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8; -*- + +from wuttjamaican import batch as mod + +try: + import sqlalchemy as sa + from wuttjamaican.db import model + from wuttjamaican.testing import DataTestCase +except ImportError: + pass +else: + + class MockBatch(model.BatchMixin, model.Base): + __tablename__ = 'testing_batch_mock' + + class MockBatchRow(model.BatchRowMixin, model.Base): + __tablename__ = 'testing_batch_mock_row' + __batch_class__ = MockBatch + + class MockBatchHandler(mod.BatchHandler): + model_class = MockBatch + + class TestBatchHandler(DataTestCase): + + def make_handler(self, **kwargs): + return MockBatchHandler(self.config, **kwargs) + + def test_model_class(self): + handler = mod.BatchHandler(self.config) + self.assertRaises(NotImplementedError, getattr, handler, 'model_class') + + def test_make_batch(self): + handler = self.make_handler() + batch = handler.make_batch(self.session) + self.assertIsInstance(batch, MockBatch) + + def test_consume_batch_id(self): + handler = self.make_handler() + + first = handler.consume_batch_id(self.session) + second = handler.consume_batch_id(self.session) + self.assertEqual(second, first + 1) + + third = handler.consume_batch_id(self.session, as_str=True) + self.assertEqual(third, f'{first + 2:08d}') + + def test_should_populate(self): + handler = self.make_handler() + batch = handler.make_batch(self.session) + self.assertFalse(handler.should_populate(batch)) + + def test_do_populate(self): + handler = self.make_handler() + batch = handler.make_batch(self.session) + # nb. coverage only; tests nothing + handler.do_populate(batch) + + def test_make_row(self): + handler = self.make_handler() + row = handler.make_row() + self.assertIsInstance(row, MockBatchRow) + + def test_add_row(self): + handler = self.make_handler() + batch = handler.make_batch(self.session) + self.session.add(batch) + row = handler.make_row() + self.assertIsNone(batch.row_count) + handler.add_row(batch, row) + self.assertEqual(batch.row_count, 1) From 3585eca65b79c0d08155edaf63131898bf33ad70 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 14 Dec 2024 19:39:02 -0600 Subject: [PATCH 3/4] fix: add basic execution methods for batch handler also logic for batch data files, and deletion --- docs/conf.py | 1 + docs/glossary.rst | 6 + pyproject.toml | 1 + src/wuttjamaican/app.py | 18 +++ src/wuttjamaican/batch.py | 233 ++++++++++++++++++++++++++++- src/wuttjamaican/db/model/batch.py | 9 +- tests/test_app.py | 8 + tests/test_batch.py | 128 ++++++++++++++++ 8 files changed, 402 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 56ce5da..8dbfd79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,6 +30,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] intersphinx_mapping = { 'alembic': ('https://alembic.sqlalchemy.org/en/latest/', None), + 'humanize': ('https://humanize.readthedocs.io/en/stable/', None), 'mako': ('https://docs.makotemplates.org/en/latest/', None), 'packaging': ('https://packaging.python.org/en/latest/', None), 'python': ('https://docs.python.org/3/', None), diff --git a/docs/glossary.rst b/docs/glossary.rst index 137aefe..58484a2 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -112,6 +112,12 @@ Glossary "inventory batch" would use another. And each "type" would be managed by its own :term:`batch handler`. + The batch type is set on the model class but is also available on + the handler: + + * :attr:`wuttjamaican.db.model.batch.BatchMixin.batch_type` + * :attr:`wuttjamaican.batch.BatchHandler.batch_type` + command A top-level command line interface for the app. Note that top-level commands don't usually "do" anything per se, and are diff --git a/pyproject.toml b/pyproject.toml index ec67103..efb2cfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ + "humanize", 'importlib-metadata; python_version < "3.10"', "importlib_resources ; python_version < '3.9'", "Mako", diff --git a/src/wuttjamaican/app.py b/src/wuttjamaican/app.py index 19766eb..102d5db 100644 --- a/src/wuttjamaican/app.py +++ b/src/wuttjamaican/app.py @@ -24,11 +24,14 @@ WuttJamaican - app handler """ +import datetime import importlib import os import sys import warnings +import humanize + from wuttjamaican.util import (load_entry_points, load_object, make_title, make_uuid, make_true_uuid, progress_loop, resource_path) @@ -714,6 +717,21 @@ class AppHandler: if value is not None: return value.strftime(self.display_format_datetime) + def render_time_ago(self, value): + """ + Return a human-friendly string, indicating how long ago + something occurred. + + Default logic uses :func:`humanize:humanize.naturaltime()` for + the rendering. + + :param value: Instance of :class:`python:datetime.datetime` or + :class:`python:datetime.timedelta`. + + :returns: Text to display. + """ + return humanize.naturaltime(value) + ############################## # getters for other handlers ############################## diff --git a/src/wuttjamaican/batch.py b/src/wuttjamaican/batch.py index ec0f708..98325ff 100644 --- a/src/wuttjamaican/batch.py +++ b/src/wuttjamaican/batch.py @@ -24,6 +24,10 @@ Batch Handlers """ +import datetime +import os +import shutil + from wuttjamaican.app import GenericHandler @@ -58,6 +62,17 @@ class BatchHandler(GenericHandler): raise NotImplementedError("You must set the 'model_class' attribute " f"for class '{self.__class__.__name__}'") + @property + def batch_type(self): + """ + Convenience property to return the :term:`batch type` which + the current handler is meant to process. + + This is effectively an alias to + :attr:`~wuttjamaican.db.model.batch.BatchMixin.batch_type`. + """ + return self.model_class.batch_type + def make_batch(self, session, progress=None, **kwargs): """ Make and return a new batch (:attr:`model_class`) instance. @@ -124,6 +139,71 @@ class BatchHandler(GenericHandler): see instead :meth:`populate()`. """ + def get_data_path(self, batch=None, filename=None, makedirs=False): + """ + Returns a path to batch data file(s). + + This can be used to return any of the following, depending on + how it's called: + + * path to root data dir for handler's :attr:`batch_type` + * path to data dir for specific batch + * path to specific filename, for specific batch + + For instance:: + + # nb. assuming batch_type = 'inventory' + batch = handler.make_batch(session, created_by=user) + + handler.get_data_path() + # => env/app/data/batch/inventory + + handler.get_data_path(batch) + # => env/app/data/batch/inventory/03/7721fe56c811ef9223743af49773a4 + + handler.get_data_path(batch, 'counts.csv') + # => env/app/data/batch/inventory/03/7721fe56c811ef9223743af49773a4/counts.csv + + :param batch: Optional batch instance. If specified, will + return path for this batch in particular. Otherwise will + return the "generic" path for handler's batch type. + + :param filename: Optional filename, in context of the batch. + If set, the returned path will include this filename. Only + relevant if ``batch`` is also specified. + + :param makedirs: Whether the folder(s) should be created, if + not already present. + + :returns: Path to root data dir for handler's batch type. + """ + # get root storage path + rootdir = self.config.get(f'{self.config.appname}.batch.storage_path') + if not rootdir: + appdir = self.app.get_appdir() + rootdir = os.path.join(appdir, 'data', 'batch') + + # get path for this batch type + path = os.path.join(rootdir, self.batch_type) + + # give more precise path, if batch was specified + if batch: + uuid = batch.uuid.hex + # nb. we use *last 2 chars* for first part of batch uuid + # path. this is because uuid7 is mostly sequential, so + # first 2 chars do not vary enough. + path = os.path.join(path, uuid[-2:], uuid[:-2]) + + # maybe create data dir + if makedirs and not os.path.exists(path): + os.makedirs(path) + + # append filename if applicable + if batch and filename: + path = os.path.join(path, filename) + + return path + def should_populate(self, batch): """ Must return true or false, indicating whether the given batch @@ -208,10 +288,161 @@ class BatchHandler(GenericHandler): to do any of the following (etc.): * fetch latest "live" data for comparison with batch input data - * calculate some data values based on the previous step + * (re-)calculate row values based on latest data * set row status based on other row attributes This method is called when the row is first added to the batch via :meth:`add_row()` - but may be called multiple times after that depending on the workflow. """ + + def why_not_execute(self, batch, user=None, **kwargs): + """ + Returns text indicating the reason (if any) that a given batch + should *not* be executed. + + By default the only reason a batch cannot be executed, is if + it has already been executed. But in some cases it should be + more restrictive; hence this method. + + A "brief but descriptive" message should be returned, which + may be displayed to the user e.g. so they understand why the + execute feature is not allowed for the batch. (There is no + need to check if batch is already executed since other logic + handles that.) + + If no text is returned, the assumption will be made that this + batch is safe to execute. + + :param batch: The batch in question; potentially eligible for + execution. + + :param user: :class:`~wuttjamaican.db.model.auth.User` who + might choose to execute the batch. + + :param \**kwargs: Execution kwargs for the batch, if known. + Should be similar to those for :meth:`execute()`. + + :returns: Text reason to prevent execution, or ``None``. + + The user interface should normally check this and if it + returns anything, that should be shown and the user should be + prevented from executing the batch. + + However :meth:`do_execute()` will also call this method, and + raise a ``RuntimeError`` if text was returned. This is done + out of safety, to avoid relying on the user interface. + """ + + def describe_execution(self, batch, user=None, **kwargs): + """ + This should return some text which briefly describes what will + happen when the given batch is executed. + + Note that Markdown is supported here, e.g.:: + + def describe_execution(self, batch, **kwargs): + return \""" + + This batch does some crazy things! + + **you cannot possibly fathom it** + + here are a few of them: + + - first + - second + - third + \""" + + Nothing is returned by default; subclass should define. + + :param batch: The batch in question; eligible for execution. + + :param user: Reference to current user who might choose to + execute the batch. + + :param \**kwargs: Execution kwargs for the batch; should be + similar to those for :meth:`execute()`. + + :returns: Markdown text describing batch execution. + """ + + def do_execute(self, batch, user, progress=None, **kwargs): + """ + Perform the execution steps for a batch. + + This first calls :meth:`why_not_execute()` to make sure this + is even allowed. + + If so, it calls :meth:`execute()` and then updates + :attr:`~wuttjamaican.db.model.batch.BatchMixin.executed` and + :attr:`~wuttjamaican.db.model.batch.BatchMixin.executed_by` on + the batch, to reflect current time+user. + + So, callers should use ``do_execute()``, and subclass should + override :meth:`execute()`. + + :param batch: The :term:`batch` to execute; instance of + :class:`~wuttjamaican.db.model.batch.BatchMixin` (among + other classes). + + :param user: :class:`~wuttjamaican.db.model.auth.User` who is + executing the batch. + + :param progress: Optional progress indicator factory. + + :param \**kwargs: Additional kwargs as needed. These are + passed as-is to :meth:`why_not_execute()` and + :meth:`execute()`. + """ + if batch.executed: + raise ValueError(f"batch has already been executed: {batch}") + + reason = self.why_not_execute(batch, user=user, **kwargs) + if reason: + raise RuntimeError(f"batch execution not allowed: {reason}") + + self.execute(batch, user=user, progress=progress, **kwargs) + batch.executed = datetime.datetime.now() + batch.executed_by = user + + def execute(self, batch, user=None, progress=None, **kwargs): + """ + Execute the given batch. + + Callers should use :meth:`do_execute()` instead, which calls + this method automatically. + + This does nothing by default; subclass must define logic. + + :param batch: A :term:`batch`; instance of + :class:`~wuttjamaican.db.model.batch.BatchMixin` (among + other classes). + + :param user: :class:`~wuttjamaican.db.model.auth.User` who is + executing the batch. + + :param progress: Optional progress indicator factory. + + :param \**kwargs: Additional kwargs which may affect the batch + execution behavior. There are none by default, but some + handlers may declare/use them. + """ + + def do_delete(self, batch, user, dry_run=False, progress=None, **kwargs): + """ + Delete the given batch entirely. + + This will delete the batch proper, all data rows, and any + files which may be associated with it. + """ + session = self.app.get_session(batch) + + # remove data files + path = self.get_data_path(batch) + if os.path.exists(path) and not dry_run: + shutil.rmtree(path) + + # remove batch proper + session.delete(batch) diff --git a/src/wuttjamaican/db/model/batch.py b/src/wuttjamaican/db/model/batch.py index 76801b8..f6ba154 100644 --- a/src/wuttjamaican/db/model/batch.py +++ b/src/wuttjamaican/db/model/batch.py @@ -47,6 +47,13 @@ class BatchMixin: handler` must be defined, which is able to process data for that :term:`batch type`. + .. attribute:: batch_type + + This is the canonical :term:`batch type` for the batch model. + + By default this will match the underlying table name for the + batch, but the model class can set it explicitly to override. + .. attribute:: __row_class__ Reference to the specific :term:`data model` class used for the @@ -194,7 +201,7 @@ class BatchMixin: ) @declared_attr - def batch_key(cls): + def batch_type(cls): return cls.__tablename__ uuid = uuid_column() diff --git a/tests/test_app.py b/tests/test_app.py index ca8c952..71a4066 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -434,6 +434,14 @@ app_title = WuttaTest dt = datetime.datetime(2024, 12, 11, 8, 30, tzinfo=datetime.timezone.utc) self.assertEqual(self.app.render_datetime(dt), '2024-12-11 08:30+0000') + def test_render_time_ago(self): + with patch.object(mod, 'humanize') as humanize: + humanize.naturaltime.return_value = 'now' + now = datetime.datetime.now() + result = self.app.render_time_ago(now) + self.assertEqual(result, 'now') + humanize.naturaltime.assert_called_once_with(now) + def test_get_person(self): people = self.app.get_people_handler() with patch.object(people, 'get_person') as get_person: diff --git a/tests/test_batch.py b/tests/test_batch.py index b29c42b..6075fd0 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -1,5 +1,8 @@ # -*- coding: utf-8; -*- +import os +from unittest.mock import patch + from wuttjamaican import batch as mod try: @@ -29,6 +32,11 @@ else: handler = mod.BatchHandler(self.config) self.assertRaises(NotImplementedError, getattr, handler, 'model_class') + def test_batch_type(self): + with patch.object(mod.BatchHandler, 'model_class', new=MockBatch): + handler = mod.BatchHandler(self.config) + self.assertEqual(handler.batch_type, 'testing_batch_mock') + def test_make_batch(self): handler = self.make_handler() batch = handler.make_batch(self.session) @@ -44,6 +52,44 @@ else: third = handler.consume_batch_id(self.session, as_str=True) self.assertEqual(third, f'{first + 2:08d}') + def test_get_data_path(self): + model = self.app.model + user = model.User(username='barney') + self.session.add(user) + + with patch.object(mod.BatchHandler, 'model_class', new=MockBatch): + handler = self.make_handler() + + # root storage (default) + with patch.object(self.app, 'get_appdir', return_value=self.tempdir): + path = handler.get_data_path() + self.assertEqual(path, os.path.join(self.tempdir, 'data', 'batch', 'testing_batch_mock')) + + # root storage (configured) + self.config.setdefault('wutta.batch.storage_path', self.tempdir) + path = handler.get_data_path() + self.assertEqual(path, os.path.join(self.tempdir, 'testing_batch_mock')) + + batch = handler.make_batch(self.session, created_by=user) + self.session.add(batch) + self.session.flush() + + # batch-specific + path = handler.get_data_path(batch) + uuid = batch.uuid.hex + final = os.path.join(uuid[-2:], uuid[:-2]) + self.assertEqual(path, os.path.join(self.tempdir, 'testing_batch_mock', final)) + + # with filename + path = handler.get_data_path(batch, 'input.csv') + self.assertEqual(path, os.path.join(self.tempdir, 'testing_batch_mock', final, 'input.csv')) + + # makedirs + path = handler.get_data_path(batch) + self.assertFalse(os.path.exists(path)) + path = handler.get_data_path(batch, makedirs=True) + self.assertTrue(os.path.exists(path)) + def test_should_populate(self): handler = self.make_handler() batch = handler.make_batch(self.session) @@ -68,3 +114,85 @@ else: self.assertIsNone(batch.row_count) handler.add_row(batch, row) self.assertEqual(batch.row_count, 1) + + def test_do_execute(self): + model = self.app.model + user = model.User(username='barney') + self.session.add(user) + + handler = self.make_handler() + batch = handler.make_batch(self.session, created_by=user) + self.session.add(batch) + self.session.flush() + + # error if execution not allowed + with patch.object(handler, 'why_not_execute', return_value="bad batch"): + self.assertRaises(RuntimeError, handler.do_execute, batch, user) + + # nb. coverage only; tests nothing + self.assertIsNone(batch.executed) + self.assertIsNone(batch.executed_by) + handler.do_execute(batch, user) + self.assertIsNotNone(batch.executed) + self.assertIs(batch.executed_by, user) + + # error if execution already happened + self.assertRaises(ValueError, handler.do_execute, batch, user) + + def test_do_delete(self): + model = self.app.model + handler = self.make_handler() + + user = model.User(username='barney') + self.session.add(user) + + # simple delete + batch = handler.make_batch(self.session, created_by=user) + self.session.add(batch) + self.session.flush() + self.assertEqual(self.session.query(MockBatch).count(), 1) + handler.do_delete(batch, user) + self.assertEqual(self.session.query(MockBatch).count(), 0) + + # delete w/ rows + batch = handler.make_batch(self.session, created_by=user) + self.session.add(batch) + for i in range(5): + row = handler.make_row() + handler.add_row(batch, row) + self.session.flush() + self.assertEqual(self.session.query(MockBatch).count(), 1) + handler.do_delete(batch, user) + self.assertEqual(self.session.query(MockBatch).count(), 0) + + # delete w/ files + self.config.setdefault('wutta.batch.storage_path', self.tempdir) + batch = handler.make_batch(self.session, created_by=user) + self.session.add(batch) + self.session.flush() + path = handler.get_data_path(batch, 'data.txt', makedirs=True) + with open(path, 'wt') as f: + f.write('foo=bar') + self.assertEqual(self.session.query(MockBatch).count(), 1) + path = handler.get_data_path(batch) + self.assertTrue(os.path.exists(path)) + handler.do_delete(batch, user) + self.assertEqual(self.session.query(MockBatch).count(), 0) + self.assertFalse(os.path.exists(path)) + + # delete w/ files (dry-run) + self.config.setdefault('wutta.batch.storage_path', self.tempdir) + batch = handler.make_batch(self.session, created_by=user) + self.session.add(batch) + self.session.flush() + path = handler.get_data_path(batch, 'data.txt', makedirs=True) + with open(path, 'wt') as f: + f.write('foo=bar') + self.assertEqual(self.session.query(MockBatch).count(), 1) + path = handler.get_data_path(batch) + self.assertTrue(os.path.exists(path)) + handler.do_delete(batch, user, dry_run=True) + # nb. batch appears missing from session even in dry-run + self.assertEqual(self.session.query(MockBatch).count(), 0) + # nb. but its files remain intact + self.assertTrue(os.path.exists(path)) From f50e0e7b99e54a032bb727551ae5e22b2c984265 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 15 Dec 2024 00:45:20 -0600 Subject: [PATCH 4/4] =?UTF-8?q?bump:=20version=200.17.1=20=E2=86=92=200.18?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 13 +++++++++++++ pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a855c37..f4371a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to WuttJamaican will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## v0.18.0 (2024-12-15) + +### Feat + +- add basic batch feature, data model and partial handler +- add basic db handler, for tracking counter values + +### Fix + +- add basic execution methods for batch handler +- add `render_date()`, `render_datetime()` methods for app handler +- add command for `wutta make-appdir` + ## v0.17.1 (2024-12-08) ### Fix diff --git a/pyproject.toml b/pyproject.toml index efb2cfa..bf74095 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "WuttJamaican" -version = "0.17.1" +version = "0.18.0" description = "Base package for Wutta Framework" readme = "README.md" authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]