From fc250a433c8ebbc362736784a7a522b8844467ce Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Dec 2025 20:03:47 -0600 Subject: [PATCH 1/6] feat: add the `import-versions` command, handler logic only works if wutta-continuum is already installed and enabled. this also rearranges some existing classes, for better consistency --- docs/api/wuttasync.cli.import_versions.rst | 6 + docs/api/wuttasync.importing.versions.rst | 6 + docs/conf.py | 7 + docs/index.rst | 2 + docs/narr/cli/builtin.rst | 21 ++ pyproject.toml | 2 +- src/wuttasync/cli/__init__.py | 3 +- src/wuttasync/cli/import_versions.py | 67 ++++ src/wuttasync/importing/__init__.py | 13 +- src/wuttasync/importing/base.py | 181 ++++++++++- src/wuttasync/importing/csv.py | 6 +- src/wuttasync/importing/handlers.py | 184 ++++++++++- src/wuttasync/importing/versions.py | 340 +++++++++++++++++++++ src/wuttasync/importing/wutta.py | 32 +- tests/cli/test_import_versions.py | 22 ++ tests/importing/test_base.py | 125 +++++++- tests/importing/test_handlers.py | 122 ++++++++ tests/importing/test_versions.py | 247 +++++++++++++++ tests/importing/test_wutta.py | 35 --- 19 files changed, 1345 insertions(+), 76 deletions(-) create mode 100644 docs/api/wuttasync.cli.import_versions.rst create mode 100644 docs/api/wuttasync.importing.versions.rst create mode 100644 src/wuttasync/cli/import_versions.py create mode 100644 src/wuttasync/importing/versions.py create mode 100644 tests/cli/test_import_versions.py create mode 100644 tests/importing/test_versions.py diff --git a/docs/api/wuttasync.cli.import_versions.rst b/docs/api/wuttasync.cli.import_versions.rst new file mode 100644 index 0000000..aeb8227 --- /dev/null +++ b/docs/api/wuttasync.cli.import_versions.rst @@ -0,0 +1,6 @@ + +``wuttasync.cli.import_versions`` +================================= + +.. automodule:: wuttasync.cli.import_versions + :members: diff --git a/docs/api/wuttasync.importing.versions.rst b/docs/api/wuttasync.importing.versions.rst new file mode 100644 index 0000000..aa970a1 --- /dev/null +++ b/docs/api/wuttasync.importing.versions.rst @@ -0,0 +1,6 @@ + +``wuttasync.importing.versions`` +================================ + +.. automodule:: wuttasync.importing.versions + :members: diff --git a/docs/conf.py b/docs/conf.py index 2b47550..7826856 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,13 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None), + "sqlalchemy": ("http://docs.sqlalchemy.org/en/latest/", None), + "sqlalchemy-continuum": ( + "https://sqlalchemy-continuum.readthedocs.io/en/latest/", + None, + ), + "sqlalchemy-utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None), + "wutta-continuum": ("https://docs.wuttaproject.org/wutta-continuum/", None), "wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None), } diff --git a/docs/index.rst b/docs/index.rst index 9eb2d93..6fe554a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,10 +73,12 @@ cf. :doc:`rattail-manual:data/sync/index`. api/wuttasync.cli api/wuttasync.cli.base api/wuttasync.cli.import_csv + api/wuttasync.cli.import_versions api/wuttasync.importing api/wuttasync.importing.base api/wuttasync.importing.csv api/wuttasync.importing.handlers api/wuttasync.importing.model + api/wuttasync.importing.versions api/wuttasync.importing.wutta api/wuttasync.util diff --git a/docs/narr/cli/builtin.rst b/docs/narr/cli/builtin.rst index 0630c94..ac6fb14 100644 --- a/docs/narr/cli/builtin.rst +++ b/docs/narr/cli/builtin.rst @@ -25,3 +25,24 @@ types may not behave as expected etc. Defined in: :mod:`wuttasync.cli.import_csv` .. program-output:: wutta import-csv --help + + +.. _wutta-import-versions: + +``wutta import-versions`` +------------------------- + +Import latest data to version tables, for the Wutta :term:`app +database`. + +The purpose of this is to ensure version tables accurately reflect +the current "live" data set, for given table(s). It is only +relevant/usable if versioning is configured and enabled. For more +on that see :doc:`wutta-continuum:index`. + +This command can check/update version tables for any versioned class +in the :term:`app model`. + +Defined in: :mod:`wuttasync.cli.import_versions` + +.. program-output:: wutta import-versions --help diff --git a/pyproject.toml b/pyproject.toml index a48b949..cff065a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] -tests = ["pylint", "pytest", "pytest-cov", "tox"] +tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"] [project.entry-points."wutta.typer_imports"] diff --git a/src/wuttasync/cli/__init__.py b/src/wuttasync/cli/__init__.py index c77a4e2..0d88ed4 100644 --- a/src/wuttasync/cli/__init__.py +++ b/src/wuttasync/cli/__init__.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -34,3 +34,4 @@ from .base import import_command, file_import_command, ImportCommandHandler # nb. must bring in all modules for discovery to work from . import import_csv +from . import import_versions diff --git a/src/wuttasync/cli/import_versions.py b/src/wuttasync/cli/import_versions.py new file mode 100644 index 0000000..f1d0481 --- /dev/null +++ b/src/wuttasync/cli/import_versions.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +See also: :ref:`wutta-import-versions` +""" + +import sys + +import rich +import typer + +from wuttjamaican.cli import wutta_typer + +from .base import import_command, ImportCommandHandler + + +@wutta_typer.command() +@import_command +def import_versions(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument + """ + Import latest data to version tables, for Wutta DB + """ + config = ctx.parent.wutta_config + app = config.get_app() + + # warn/exit if libs are not installed + try: + import wutta_continuum # pylint: disable=import-outside-toplevel,unused-import + except ImportError: # pragma: no cover + rich.print( + "\n\t[bold yellow]Wutta-Continum is not installed![/bold yellow]\n" + "\n\tIf you want it, run: pip install Wutta-Continuum\n" + ) + sys.exit(1) + + # warn/exit if feature disabled + if not app.continuum_is_enabled(): # pragma: no cover + rich.print( + "\n\t[bold yellow]Wutta-Continum is not enabled![/bold yellow]\n" + "\n\tIf you want it, see: https://docs.wuttaproject.org/wutta-continuum/\n" + ) + sys.exit(1) + + handler = ImportCommandHandler( + config, import_handler="wuttasync.importing.versions:FromWuttaToVersions" + ) + handler.run(ctx.params) diff --git a/src/wuttasync/importing/__init__.py b/src/wuttasync/importing/__init__.py index 03a421f..545cbb9 100644 --- a/src/wuttasync/importing/__init__.py +++ b/src/wuttasync/importing/__init__.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -32,7 +32,7 @@ And some :term:`import handler` base classes: * :class:`~wuttasync.importing.handlers.ImportHandler` * :class:`~wuttasync.importing.handlers.FromFileHandler` * :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler` -* :class:`~wuttasync.importing.wutta.ToWuttaHandler` +* :class:`~wuttasync.importing.handlers.ToWuttaHandler` And some :term:`importer` base classes: @@ -42,7 +42,12 @@ And some :term:`importer` base classes: * :class:`~wuttasync.importing.model.ToWutta` """ -from .handlers import Orientation, ImportHandler, FromFileHandler, ToSqlalchemyHandler +from .handlers import ( + Orientation, + ImportHandler, + FromFileHandler, + ToSqlalchemyHandler, + ToWuttaHandler, +) from .base import Importer, FromFile, ToSqlalchemy from .model import ToWutta -from .wutta import ToWuttaHandler diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py index 629ead6..ca0718e 100644 --- a/src/wuttasync/importing/base.py +++ b/src/wuttasync/importing/base.py @@ -184,6 +184,19 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- :meth:`get_target_cache()`. """ + default_keys = None + """ + In certain edge cases, the importer class must declare its key + list without using :attr:`keys`. + + (As of now this only happens with + :class:`~wuttasync.importing.versions.FromWuttaToVersions` which + must dynamically create importer classes.) + + If applicable, this value is used as fallback for + :meth:`get_keys()`. + """ + max_create = None max_update = None max_delete = None @@ -323,19 +336,54 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- def get_keys(self): """ - Must return the key field(s) for use with import/export. + Retrieve the list of key field(s) for use with import/export. + The result is cached, so the key list is only calculated once. + + Many importers have just one key field, but we always assume a + key *list* - so this often is a list with just one field. All fields in this list should also be found in the output for :meth:`get_fields()`. + Many importers will declare this via :attr:`keys` (or + :attr:`key`) static attribute:: + + class SprocketImporter(Importer): + + # nb. all these examples work the same + + # 'keys' is the preferred attribute + keys = ("sprocket_id",) # <-- the "canonical" way + keys = ["sprocket_id"] + keys = "sprocket_id" + + # 'key' is not preferred, but works + key = ("sprocket_id",) + key = "sprocket_id" + + If neither ``keys`` nor ``key`` is set, as a special case + :attr:`default_keys` is used if set. + + If no keys were declared, the list is inspected from the model + class via + :func:`sqlalchemy-utils:sqlalchemy_utils.functions.get_primary_keys()`. + + In any case, the determination is made only once. This method + also *sets* :attr:`keys` on the instance, so it will return + that as-is for subsequent calls. + :returns: List of "key" field names. """ keys = None + # nb. prefer 'keys' but use 'key' as fallback if "keys" in self.__dict__: keys = self.__dict__["keys"] elif "key" in self.__dict__: keys = self.__dict__["key"] + else: + keys = self.default_keys + if keys: if isinstance(keys, str): keys = self.config.parse_list(keys) @@ -1271,10 +1319,139 @@ class FromFile(Importer): self.input_file.close() +class QueryWrapper: + """ + Simple wrapper for a SQLAlchemy query, to make it sort of behave + so that an importer can treat it as a data record list. + + :param query: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance + """ + + def __init__(self, query): + self.query = query + + def __len__(self): + try: + return len(self.query) + except TypeError: + return self.query.count() + + def __iter__(self): + return iter(self.query) + + +class FromSqlalchemy(Importer): # pylint: disable=abstract-method + """ + Base class for importer/exporter using SQL/ORM query as data + source. + + Subclass should define :attr:`source_model_class` in which case + the source query is automatic. And/or override + :meth:`get_source_query()` to customize. + + See also :class:`FromSqlalchemyMirror` and :class:`ToSqlalchemy`. + """ + + source_model_class = None + """ + Reference to the :term:`data model` class representing the source. + + This normally is a SQLAlchemy mapped class, e.g. + :class:`~wuttjamaican:wuttjamaican.db.model.base.Person` for + exporting from the Wutta People table. + """ + + source_session = None + """ + Reference to the open :term:`db session` for the data source. + + The importer must be given this reference when instantiated by the + :term:`import handler`. This is handled automatically if using + :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`. + """ + + def get_source_objects(self): + """ + This method is responsible for fetching "raw" (non-normalized) + records from data source. + + (See also the parent method docs for + :meth:`~wuttasync.importing.base.Importer.get_source_objects()`.) + + It calls :meth:`get_source_query()` and then wraps that in a + :class:`QueryWrapper`, which is then returned. + + Note that this method does not technically "retrieve" records + from the query; that happens automatically later. + + :returns: :class:`QueryWrapper` for the source query + """ + query = self.get_source_query() + return QueryWrapper(query) + + def get_source_query(self): + """ + This returns the SQL/ORM query used to fetch source + data. It is called from :meth:`get_source_objects()`. + + Default logic just makes a simple ``SELECT * FROM TABLE`` kind + of query. Subclass can override as needed. + + :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance + """ + return self.source_session.query(self.source_model_class) + + +class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method + """ + Special base class for when the source and target are effectively + mirrored, and can each be represented by the same :term:`data + model`. + + The assumption is that SQLAlchemy ORM is used on both sides, even + though this base class only defines the source side (it inherits + from :class:`FromSqlalchemy`). + + There are two main use cases for this: + + * sync between app nodes + * sync version tables + + When 2 app nodes are synced, the source and target are "the same" + in a schema sense, e.g. ``sprockets on node 01 => sprockets on + node 02``. + + When version tables are synced, the same schema can be used for + the "live" table and the "version" table, e.g. ``sprockets => + sprocket versions``. + """ + + @property + def source_model_class(self): + """ + This returns the :attr:`~Importer.model_class` since source + and target must share common schema. + """ + return self.model_class + + def normalize_source_object(self, obj): + """ + Since source/target share schema, there should be no tricky + normalization involved. + + This calls :meth:`~Importer.normalize_target_object()` since + that logic should already be defined. This ensures the same + normalization is used on both sides. + """ + return self.normalize_target_object(obj) + + class ToSqlalchemy(Importer): """ Base class for importer/exporter which uses SQLAlchemy ORM on the target side. + + See also :class:`FromSqlalchemy`. """ caches_target = True @@ -1312,6 +1489,8 @@ class ToSqlalchemy(Importer): Returns an ORM query suitable to fetch existing objects from the target side. This is called from :meth:`get_target_objects()`. + + :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance """ return self.target_session.query(self.model_class) diff --git a/src/wuttasync/importing/csv.py b/src/wuttasync/importing/csv.py index ab0bf21..9190099 100644 --- a/src/wuttasync/importing/csv.py +++ b/src/wuttasync/importing/csv.py @@ -38,8 +38,7 @@ from wuttjamaican.db.util import make_topo_sortkey, UUID from wuttjamaican.util import parse_bool from .base import FromFile -from .handlers import FromFileHandler -from .wutta import ToWuttaHandler +from .handlers import FromFileHandler, ToWuttaHandler from .model import ToWutta @@ -239,6 +238,8 @@ class FromCsvToSqlalchemyHandlerMixin: """ raise NotImplementedError + # TODO: pylint (correctly) flags this as duplicate code, matching + # on the wuttasync.importing.versions module - should fix? def define_importers(self): """ This mixin overrides typical (manual) importer definition, and @@ -252,6 +253,7 @@ class FromCsvToSqlalchemyHandlerMixin: importers = {} model = self.get_target_model() + # pylint: disable=duplicate-code # mostly try to make an importer for every data model for name in dir(model): cls = getattr(model, name) diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index e9c6ac3..c1f7595 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -209,6 +209,20 @@ class ImportHandler(GenericHandler): """ Returns the display title for the data source. + By default this returns :attr:`source_key`, but this can be + overriden by class attribute. + + Base class can define ``generic_source_title`` to provide a + new default:: + + class FromExcelHandler(ImportHandler): + generic_source_title = "Excel File" + + Subclass can define ``source_title`` to be explicit:: + + class FromExcelToWutta(FromExcelHandler, ToWuttaHandler): + source_title = "My Spreadsheet" + See also :meth:`get_title()` and :meth:`get_target_title()`. """ if hasattr(self, "source_title"): @@ -221,6 +235,20 @@ class ImportHandler(GenericHandler): """ Returns the display title for the data target. + By default this returns :attr:`target_key`, but this can be + overriden by class attribute. + + Base class can define ``generic_target_title`` to provide a + new default:: + + class ToExcelHandler(ImportHandler): + generic_target_title = "Excel File" + + Subclass can define ``target_title`` to be explicit:: + + class FromWuttaToExcel(FromWuttaHandler, ToExcelHandler): + target_title = "My Spreadsheet" + See also :meth:`get_title()` and :meth:`get_source_title()`. """ if hasattr(self, "target_title"): @@ -538,9 +566,129 @@ class FromFileHandler(ImportHandler): super().process_data(*keys, **kwargs) +class FromSqlalchemyHandler(ImportHandler): + """ + Base class for import/export handlers using SQLAlchemy ORM (DB) as + data source. + + This is meant to be used with importers/exporters which inherit + from :class:`~wuttasync.importing.base.FromSqlalchemy`. It will + set the + :attr:`~wuttasync.importing.base.FromSqlalchemy.source_session` + attribute when making them; cf. :meth:`get_importer_kwargs()`. + + This is the base class for :class:`FromWuttaHandler`, but can be + used with any database. + + See also :class:`ToSqlalchemyHandler`. + """ + + source_session = None + """ + Reference to the :term:`db session` for data source. + + This will be ``None`` unless a transaction is running. + """ + + def begin_source_transaction(self): + """ + This calls :meth:`make_source_session()` and assigns the + result to :attr:`source_session`. + """ + self.source_session = self.make_source_session() + + def commit_source_transaction(self): + """ + This commits and closes :attr:`source_session`. + """ + self.source_session.commit() + self.source_session.close() + self.source_session = None + + def rollback_source_transaction(self): + """ + This rolls back, then closes :attr:`source_session`. + """ + self.source_session.rollback() + self.source_session.close() + self.source_session = None + + def make_source_session(self): + """ + Make and return a new :term:`db session` for the data source. + + Default logic is not implemented; subclass must override. + + :returns: :class:`~sqlalchemy.orm.Session` instance + """ + raise NotImplementedError + + def get_importer_kwargs(self, key, **kwargs): + """ + This modifies the new importer kwargs to add: + + * ``source_session`` - reference to :attr:`source_session` + + See also docs for parent method, + :meth:`~ImportHandler.get_importer_kwargs()`. + """ + kwargs = super().get_importer_kwargs(key, **kwargs) + kwargs["source_session"] = self.source_session + return kwargs + + +class FromWuttaHandler(FromSqlalchemyHandler): + """ + Handler for import/export which uses Wutta ORM (:term:`app + database`) as data source. + + This inherits from :class:`FromSqlalchemyHandler`. + + See also :class:`ToWuttaHandler`. + """ + + source_key = "wutta" + "" # nb. suppress docs + + def get_source_title(self): + """ + This overrides default logic to use + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.get_title()` + as the default value. + + Subclass can still define + :attr:`~wuttasync.importing.handlers.ImportHandler.source_title` + (or + :attr:`~wuttasync.importing.handlers.ImportHandler.generic_source_title`) + to customize. + + See also docs for parent method: + :meth:`~wuttasync.importing.handlers.ImportHandler.get_source_title()` + """ + if hasattr(self, "source_title"): + return self.source_title + if hasattr(self, "generic_source_title"): + return self.generic_source_title + return self.app.get_title() + + def make_source_session(self): + """ + This calls + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` + and returns it. + """ + return self.app.make_session() + + class ToSqlalchemyHandler(ImportHandler): """ - Handler for import/export which targets a SQLAlchemy ORM (DB). + Base class for import/export handlers which target a SQLAlchemy + ORM (DB). + + This is the base class for :class:`ToWuttaHandler`, but can be + used with any database. + + See also :class:`FromSqlalchemyHandler`. """ target_session = None @@ -591,3 +739,37 @@ class ToSqlalchemyHandler(ImportHandler): kwargs = super().get_importer_kwargs(key, **kwargs) kwargs.setdefault("target_session", self.target_session) return kwargs + + +class ToWuttaHandler(ToSqlalchemyHandler): + """ + Handler for import/export which targets Wutta ORM (:term:`app + database`). + + This inherits from :class:`ToSqlalchemyHandler`. + + See also :class:`FromWuttaHandler`. + """ + + target_key = "wutta" + "" # nb. suppress docs + + def get_target_title(self): # pylint: disable=empty-docstring + """ """ + # nb. we override parent to use app title as default + if hasattr(self, "target_title"): + return self.target_title + if hasattr(self, "generic_target_title"): + return self.generic_target_title + return self.app.get_title() + + def make_target_session(self): + """ + Call + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` + and return the result. + + :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session` + instance. + """ + return self.app.make_session() diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py new file mode 100644 index 0000000..53c25fa --- /dev/null +++ b/src/wuttasync/importing/versions.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Importing Versions + +This is a special type of import, only relevant when data versioning +is enabled. + +See the handler class for more info: :class:`FromWuttaToVersions` +""" + +from collections import OrderedDict + +from sqlalchemy_utils.functions import get_primary_keys + +from wuttjamaican.db.util import make_topo_sortkey + +from .handlers import FromWuttaHandler, ToWuttaHandler +from .wutta import FromWuttaMirror +from .model import ToWutta + + +class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler): + """ + Handler for Wutta -> Versions import. + + The purpose of this is to ensure version tables accurately reflect + the current "live" data set, for given table(s). It is only + relevant/usable if versioning is configured and enabled. For more + on that see :doc:`wutta-continuum:index`. + + For a given import model, the source is the "live" table, target + is the "version" table - both in the same :term:`app database`. + + When reading data from the target side, it only grabs the "latest" + (valid) version record for each comparison to source. + + When changes are needed, instead of updating the existing version + record, it always writes a new version record. + + This handler will dynamically create importers for all versioned + models in the :term:`app model`; see + :meth:`make_importer_factory()`. + """ + + target_key = "versions" + target_title = "Versions" + + continuum_uow = None + """ + Reference to the + :class:`sqlalchemy-continuum:`sqlalchemy_continuum.UnitOfWork` + created (by the SQLAlchemy-Continuum ``versioning_manager``) when + the transaction begins. + + See also :attr:`continuum_txn` and + :meth:`begin_target_transaction()`. + """ + + continuum_txn = None + """ + Reference to the SQLAlchemy-Continuum ``transaction`` record, to + which any new version records will associate (if needed). + + This transaction will track the effective user responsible for + the change(s), their client IP, and timestamp. + + This reference is passed along to the importers as well (as + :attr:`~FromWuttaToVersionBase.continuum_txn`) via + :meth:`get_importer_kwargs()`. + + See also :attr:`continuum_uow`. + """ + + def begin_target_transaction(self): + # pylint: disable=line-too-long + """ + In addition to normal logic, this does some setup for + SQLAlchemy-Continuum: + + It establishes a "unit of work" by calling + :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.VersioningManager.unit_of_work()`, + assigning the result to :attr:`continuum_uow`. + + It then calls + :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()` + and assigns that to :attr:`continuum_txn`. + + See also docs for parent method: + :meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()` + """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + super().begin_target_transaction() + + self.continuum_uow = continuum.versioning_manager.unit_of_work( + self.target_session + ) + self.continuum_txn = self.continuum_uow.create_transaction(self.target_session) + + def get_importer_kwargs(self, key, **kwargs): + """ + This modifies the new importer kwargs to add: + + * ``continuum_txn`` - reference to :attr:`continuum_txn` + + See also docs for parent method: + :meth:`~wuttasync.importing.handlers.ImportHandler.get_importer_kwargs()` + """ + kwargs = super().get_importer_kwargs(key, **kwargs) + kwargs["continuum_txn"] = self.continuum_txn + return kwargs + + # TODO: pylint (correctly) flags this as duplicate code, matching + # on the wuttasync.importing.csv module - should fix? + def define_importers(self): + """ + This overrides typical (manual) importer definition, instead + generating importers for all versioned models. + + It will inspect the :term:`app model` and call + :meth:`make_importer_factory()` for each model found, keeping + only the valid importers. + + See also the docs for parent method: + :meth:`~wuttasync.importing.handlers.ImportHandler.define_importers()` + """ + model = self.app.model + importers = {} + + # pylint: disable=duplicate-code + # mostly try to make an importer for every data model + for name in dir(model): + cls = getattr(model, name) + if ( + isinstance(cls, type) + and issubclass(cls, model.Base) + and cls is not model.Base + ): + # only keep "good" importers, i.e. for versioned models + if factory := self.make_importer_factory(cls, name): + importers[name] = factory + + # sort importers according to schema topography + topo_sortkey = make_topo_sortkey(model) + importers = OrderedDict( + [(name, importers[name]) for name in sorted(importers, key=topo_sortkey)] + ) + + return importers + + def make_importer_factory(self, model_class, name): + """ + Try to generate a new :term:`importer` class for the given + :term:`data model`. This is called by + :meth:`define_importers()`. + + If the provided ``model_class`` is not versioned, this will + fail and return ``None``. + + For a versioned model, the new importer class will inherit + from :class:`FromWuttaToVersionBase`. + + Its (target) + :attr:`~wuttasync.importing.base.Importer.model_class` will be + set to the **version** model. + + Its + :attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class` + will be set to the **normal** model. + + :param model_class: A (normal, not version) data model class. + + :param name: The "model name" for the importer. New class + name will be based on this, so e.g. ``Widget`` model name + becomes ``WidgetImporter`` class name. + + :returns: The new class, or ``None`` + """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + try: + version_class = continuum.version_class(model_class) + except continuum.exc.ClassNotVersioned: + return None + + return type( + f"{name}Importer", + (FromWuttaToVersionBase,), + { + "source_model_class": model_class, + "model_class": version_class, + "default_keys": list(get_primary_keys(model_class)), + }, + ) + + +class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): + """ + Base importer class for Wutta -> Versions. + + This imports from + :class:`~wuttasync.importing.wutta.FromWuttaMirror` and + :class:`~wuttasync.importing.model.ToWutta`. + + The import handler will dynamically generate importers using this + base class; see + :meth:`~FromWuttaToVersions.make_importer_factory()`. + """ + + continuum_txn = None + """ + Reference to the handler's attribute of the same name: + :attr:`~FromWuttaToVersions.continuum_txn` + + This is the SQLAlchemy-Continuum ``transaction`` record, to which + any new version records will associate (if needed). + + This transaction will track the effective user responsible for + the change(s), their client IP, and timestamp. + """ + + def get_simple_fields(self): # pylint: disable=empty-docstring + """ """ + fields = super().get_simple_fields() + unwanted = ["transaction_id", "operation_type", "end_transaction_id"] + fields = [field for field in fields if field not in unwanted] + return fields + + def get_target_query(self, source_data=None): + """ + This modifies the normal query to ensure we only get the + "latest valid" version for each record, for comparison to + source. + + .. note:: + + In some cases, it still may be possible for multiple + "latest" versions to match for a given record. This means + inconsistent data; a warning should be logged if so, and + you must track it down... + + See also docs for parent method: + :meth:`~wuttasync.importing.base.ToSqlalchemy.get_target_query()` + """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + # pylint: disable=singleton-comparison + return ( + self.target_session.query(self.model_class) + .filter(self.model_class.end_transaction_id == None) + .filter(self.model_class.operation_type != continuum.Operation.DELETE) + ) + + def normalize_target_object(self, obj): # pylint: disable=empty-docstring + """ """ + data = super().normalize_target_object(obj) + + # we want to add the original version object to normalized + # data, so we can access it later for updating if needed. but + # this method is called for *both* sides (source+target) since + # this is a "mirrored" importer. so we must check the type + # and only cache true versions, ignore "normal" objects. + if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type + obj, self.model_class + ): + data["_version"] = obj + + return data + + def make_version( # pylint: disable=missing-function-docstring + self, source_data, operation_type + ): + key = self.get_record_key(source_data) + with self.target_session.no_autoflush: + version = self.make_empty_object(key) + self.populate(version, source_data) + version.transaction = self.continuum_txn + version.operation_type = operation_type + self.target_session.add(version) + return version + + def populate(self, obj, data): # pylint: disable=missing-function-docstring + keys = self.get_keys() + for field in self.get_simple_fields(): + if field not in keys and field in data and field in self.fields: + setattr(obj, field, data[field]) + + def create_target_object(self, key, source_data): # pylint: disable=empty-docstring + """ """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + return self.make_version(source_data, continuum.Operation.INSERT) + + def update_target_object( # pylint: disable=empty-docstring + self, obj, source_data, target_data=None + ): + """ """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + # when we "update" it always involves making a *new* version + # record. but that requires actually updating the "previous" + # version to indicate the new version's transaction. + prev_version = target_data.pop("_version") + prev_version.end_transaction_id = self.continuum_txn.id + + return self.make_version(source_data, continuum.Operation.UPDATE) + + def delete_target_object(self, obj): # pylint: disable=empty-docstring + """ """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + # nb. `obj` here is the existing/old version record; we update + # it to indicate the new version's transaction. + obj.end_transaction_id = self.continuum_txn.id + + # add new "DELETE" version record. values should be the same as + # for "previous" (existing/old) version. + source_data = self.normalize_target_object(obj) + return self.make_version(source_data, continuum.Operation.DELETE) diff --git a/src/wuttasync/importing/wutta.py b/src/wuttasync/importing/wutta.py index 9de4822..882f7df 100644 --- a/src/wuttasync/importing/wutta.py +++ b/src/wuttasync/importing/wutta.py @@ -21,37 +21,13 @@ # ################################################################################ """ -Wutta ⇄ Wutta import/export +Wutta → Wutta import/export """ -from .handlers import ToSqlalchemyHandler +from .base import FromSqlalchemyMirror -class ToWuttaHandler(ToSqlalchemyHandler): +class FromWuttaMirror(FromSqlalchemyMirror): # pylint: disable=abstract-method """ - Handler for import/export which targets Wutta ORM (:term:`app - database`). + Base class for Wutta -> Wutta data importers. """ - - target_key = "wutta" - "" # nb. suppress docs - - def get_target_title(self): # pylint: disable=empty-docstring - """ """ - # nb. we override parent to use app title as default - if hasattr(self, "target_title"): - return self.target_title - if hasattr(self, "generic_target_title"): - return self.generic_target_title - return self.app.get_title() - - def make_target_session(self): - """ - Call - :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` - and return the result. - - :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session` - instance. - """ - return self.app.make_session() diff --git a/tests/cli/test_import_versions.py b/tests/cli/test_import_versions.py new file mode 100644 index 0000000..ea1617d --- /dev/null +++ b/tests/cli/test_import_versions.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8; -*- + +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from wuttasync.cli import import_versions as mod, ImportCommandHandler + + +class TestImportCsv(TestCase): + + def test_basic(self): + params = { + "models": [], + "create": True, + "update": True, + "delete": False, + "dry_run": True, + } + ctx = MagicMock(params=params) + with patch.object(ImportCommandHandler, "run") as run: + mod.import_versions(ctx) + run.assert_called_once_with(params) diff --git a/tests/importing/test_base.py b/tests/importing/test_base.py index 08c37a2..c920ed6 100644 --- a/tests/importing/test_base.py +++ b/tests/importing/test_base.py @@ -2,6 +2,8 @@ from unittest.mock import patch +from sqlalchemy import orm + from wuttjamaican.testing import DataTestCase from wuttasync.importing import base as mod, ImportHandler, Orientation @@ -78,13 +80,31 @@ class TestImporter(DataTestCase): def test_get_keys(self): model = self.app.model + + # nb. get_keys() will cache the return value, so must + # re-create importer for each test + + # keys inspected from model by default imp = self.make_importer(model_class=model.Setting) self.assertEqual(imp.get_keys(), ["name"]) - with patch.multiple(imp, create=True, key="value"): - self.assertEqual(imp.get_keys(), ["value"]) - with patch.multiple(imp, create=True, keys=["foo", "bar"]): + imp = self.make_importer(model_class=model.User) + self.assertEqual(imp.get_keys(), ["uuid"]) + + # class may define 'keys' + imp = self.make_importer(model_class=model.User) + with patch.object(imp, "keys", new=["foo", "bar"], create=True): self.assertEqual(imp.get_keys(), ["foo", "bar"]) + # class may define 'key' + imp = self.make_importer(model_class=model.User) + with patch.object(imp, "key", new="whatever", create=True): + self.assertEqual(imp.get_keys(), ["whatever"]) + + # class may define 'default_keys' + imp = self.make_importer(model_class=model.User) + with patch.object(imp, "default_keys", new=["baz", "foo"]): + self.assertEqual(imp.get_keys(), ["baz", "foo"]) + def test_process_data(self): model = self.app.model imp = self.make_importer( @@ -651,6 +671,105 @@ class TestFromFile(DataTestCase): close.assert_called_once_with() +class TestQueryWrapper(DataTestCase): + + def test_basic(self): + model = self.app.model + + p1 = model.Person(full_name="John Doe") + self.session.add(p1) + p2 = model.Person(full_name="Jane Doe") + self.session.add(p2) + self.session.commit() + + # cannot get count via len(query), must use query.count() + query = self.session.query(model.Person) + self.assertEqual(query.count(), 2) + self.assertRaises(TypeError, len, query) + + # but can use len(wrapper) + wrapper = mod.QueryWrapper(query) + self.assertEqual(len(wrapper), 2) + + # iter(wrapper) should work too + people = [p for p in wrapper] + self.assertEqual(people, [p1, p2]) + people = [p for p in iter(wrapper)] + self.assertEqual(people, [p1, p2]) + people = [p for p in list(wrapper)] + self.assertEqual(people, [p1, p2]) + + +class TestFromSqlalchemy(DataTestCase): + + def setUp(self): + self.setup_db() + self.handler = ImportHandler(self.config) + + def make_importer(self, **kwargs): + kwargs.setdefault("handler", self.handler) + return mod.FromSqlalchemy(self.config, **kwargs) + + def test_get_source_query(self): + model = self.app.model + imp = self.make_importer( + source_model_class=model.Upgrade, source_session=self.session + ) + query = imp.get_source_query() + self.assertIsInstance(query, orm.Query) + self.assertEqual(len(query.selectable.froms), 1) + table = query.selectable.froms[0] + self.assertEqual(table.name, "upgrade") + + def test_get_source_objects(self): + model = self.app.model + + user1 = model.User(username="fred") + self.session.add(user1) + user2 = model.User(username="bettie") + self.session.add(user2) + self.session.commit() + + imp = self.make_importer( + source_model_class=model.User, source_session=self.session + ) + result = imp.get_source_objects() + self.assertIsInstance(result, mod.QueryWrapper) + self.assertEqual(len(result), 2) + self.assertEqual(list(result), [user1, user2]) + + +class TestFromSqlalchemyMirror(DataTestCase): + + def setUp(self): + self.setup_db() + self.handler = ImportHandler(self.config) + + def make_importer(self, **kwargs): + kwargs.setdefault("handler", self.handler) + return mod.FromSqlalchemyMirror(self.config, **kwargs) + + def test_source_model_class(self): + model = self.app.model + + # source_model_class will mirror model_class + imp = self.make_importer(model_class=model.Upgrade) + self.assertIs(imp.model_class, model.Upgrade) + self.assertIs(imp.source_model_class, model.Upgrade) + + def test_normalize_source_object(self): + model = self.app.model + imp = self.make_importer(model_class=model.Upgrade) + upgrade = model.Upgrade() + + # normalize_source_object() should invoke normalize_target_object() + with patch.object(imp, "normalize_target_object") as normalize_target_object: + normalize_target_object.return_value = 42 + result = imp.normalize_source_object(upgrade) + self.assertEqual(result, 42) + normalize_target_object.assert_called_once_with(upgrade) + + class TestToSqlalchemy(DataTestCase): def setUp(self): diff --git a/tests/importing/test_handlers.py b/tests/importing/test_handlers.py index 9bd0157..a6df032 100644 --- a/tests/importing/test_handlers.py +++ b/tests/importing/test_handlers.py @@ -213,6 +213,97 @@ class TestFromFileHandler(DataTestCase): process_data.assert_called_once_with(input_file_dir=self.tempdir) +class TestFromSqlalchemyHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.FromSqlalchemyHandler(self.config, **kwargs) + + def test_make_source_session(self): + handler = self.make_handler() + self.assertRaises(NotImplementedError, handler.make_source_session) + + def test_begin_source_transaction(self): + handler = self.make_handler() + self.assertIsNone(handler.source_session) + with patch.object(handler, "make_source_session", return_value=self.session): + handler.begin_source_transaction() + self.assertIs(handler.source_session, self.session) + + def test_commit_source_transaction(self): + model = self.app.model + handler = self.make_handler() + handler.source_session = self.session + self.assertEqual(self.session.query(model.User).count(), 0) + + # nb. do not commit this yet + user = model.User(username="fred") + self.session.add(user) + + self.assertTrue(self.session.in_transaction()) + self.assertIn(user, self.session) + handler.commit_source_transaction() + self.assertIsNone(handler.source_session) + self.assertFalse(self.session.in_transaction()) + self.assertNotIn(user, self.session) # hm, surprising? + self.assertEqual(self.session.query(model.User).count(), 1) + + def test_rollback_source_transaction(self): + model = self.app.model + handler = self.make_handler() + handler.source_session = self.session + self.assertEqual(self.session.query(model.User).count(), 0) + + # nb. do not commit this yet + user = model.User(username="fred") + self.session.add(user) + + self.assertTrue(self.session.in_transaction()) + self.assertIn(user, self.session) + handler.rollback_source_transaction() + self.assertIsNone(handler.source_session) + self.assertFalse(self.session.in_transaction()) + self.assertNotIn(user, self.session) + self.assertEqual(self.session.query(model.User).count(), 0) + + def test_get_importer_kwargs(self): + handler = self.make_handler() + handler.source_session = self.session + kw = handler.get_importer_kwargs("User") + self.assertIn("source_session", kw) + self.assertIs(kw["source_session"], self.session) + + +class TestFromWuttaHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.FromWuttaHandler(self.config, **kwargs) + + def test_get_source_title(self): + handler = self.make_handler() + + # uses app title by default + self.config.setdefault("wutta.app_title", "What About This") + self.assertEqual(handler.get_source_title(), "What About This") + + # or generic default if present + handler.generic_source_title = "WHATABOUTTHIS" + self.assertEqual(handler.get_source_title(), "WHATABOUTTHIS") + + # but prefer specific title if present + handler.source_title = "what_about_this" + self.assertEqual(handler.get_source_title(), "what_about_this") + + def test_make_source_session(self): + handler = self.make_handler() + + # makes "new" (mocked in our case) app session + with patch.object(self.app, "make_session") as make_session: + make_session.return_value = self.session + session = handler.make_source_session() + make_session.assert_called_once_with() + self.assertIs(session, self.session) + + class TestToSqlalchemyHandler(DataTestCase): def make_handler(self, **kwargs): @@ -256,3 +347,34 @@ class TestToSqlalchemyHandler(DataTestCase): kw = handler.get_importer_kwargs("Setting") self.assertIn("target_session", kw) self.assertIs(kw["target_session"], self.session) + + +class TestToWuttaHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.ToWuttaHandler(self.config, **kwargs) + + def test_get_target_title(self): + handler = self.make_handler() + + # uses app title by default + self.config.setdefault("wutta.app_title", "What About This") + self.assertEqual(handler.get_target_title(), "What About This") + + # or generic default if present + handler.generic_target_title = "WHATABOUTTHIS" + self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS") + + # but prefer specific title if present + handler.target_title = "what_about_this" + self.assertEqual(handler.get_target_title(), "what_about_this") + + def test_make_target_session(self): + handler = self.make_handler() + + # makes "new" (mocked in our case) app session + with patch.object(self.app, "make_session") as make_session: + make_session.return_value = self.session + session = handler.make_target_session() + make_session.assert_called_once_with() + self.assertIs(session, self.session) diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py new file mode 100644 index 0000000..2067f93 --- /dev/null +++ b/tests/importing/test_versions.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8; -*- + +from sqlalchemy import orm +import sqlalchemy_continuum as continuum + +from wuttjamaican.util import make_true_uuid +from wutta_continuum.testing import VersionTestCase + +from wuttasync.importing import versions as mod, Importer + + +class TestFromWuttaToVersions(VersionTestCase): + + def make_handler(self, **kwargs): + return mod.FromWuttaToVersions(self.config, **kwargs) + + def test_begin_target_transaction(self): + model = self.app.model + txncls = continuum.transaction_class(model.User) + + handler = self.make_handler() + self.assertIsNone(handler.continuum_uow) + self.assertIsNone(handler.continuum_txn) + + handler.begin_target_transaction() + self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork) + self.assertIsInstance(handler.continuum_txn, txncls) + + def test_get_importer_kwargs(self): + handler = self.make_handler() + handler.begin_target_transaction() + + kw = handler.get_importer_kwargs("User") + self.assertIn("continuum_txn", kw) + self.assertIs(kw["continuum_txn"], handler.continuum_txn) + + def test_make_importer_factory(self): + model = self.app.model + handler = self.make_handler() + + # versioned class + factory = handler.make_importer_factory(model.User, "User") + self.assertTrue(issubclass(factory, mod.FromWuttaToVersionBase)) + self.assertIs(factory.source_model_class, model.User) + self.assertIs(factory.model_class, continuum.version_class(model.User)) + + # non-versioned + factory = handler.make_importer_factory(model.Upgrade, "Upgrade") + self.assertIsNone(factory) + + def test_define_importers(self): + handler = self.make_handler() + + importers = handler.define_importers() + self.assertIn("User", importers) + self.assertIn("Person", importers) + self.assertNotIn("Upgrade", importers) + + +class UserImporter(mod.FromWuttaToVersionBase): + + @property + def model_class(self): + model = self.app.model + return model.User + + +class TestFromWuttaToVersionBase(VersionTestCase): + + def make_importer(self, model_class=None, **kwargs): + imp = mod.FromWuttaToVersionBase(self.config, **kwargs) + if model_class: + imp.model_class = model_class + return imp + + def test_get_simple_fields(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + # first confirm what a "normal" importer would do + imp = Importer(self.config, model_class=vercls) + fields = imp.get_simple_fields() + self.assertIn("username", fields) + self.assertIn("person_uuid", fields) + self.assertIn("transaction_id", fields) + self.assertIn("operation_type", fields) + self.assertIn("end_transaction_id", fields) + + # now test what the "version" importer does + imp = self.make_importer(model_class=vercls) + fields = imp.get_simple_fields() + self.assertIn("username", fields) + self.assertIn("person_uuid", fields) + self.assertNotIn("transaction_id", fields) + self.assertNotIn("operation_type", fields) + self.assertNotIn("end_transaction_id", fields) + + def test_get_target_query(self): + model = self.app.model + vercls = continuum.version_class(model.User) + imp = self.make_importer(model_class=vercls, target_session=self.session) + + # TODO: not sure what else to test here.. + query = imp.get_target_query() + self.assertIsInstance(query, orm.Query) + + def test_normalize_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + imp = self.make_importer(model_class=vercls) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + version = user.versions[0] + + # version object should be embedded in data dict + data = imp.normalize_target_object(version) + self.assertIsInstance(data, dict) + self.assertIn("_version", data) + self.assertIs(data["_version"], version) + + # but normal object is not embedded + data = imp.normalize_target_object(user) + self.assertIsInstance(data, dict) + self.assertNotIn("_version", data) + + def test_make_version(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + data = {"uuid": user.uuid, "username": "freddie"} + version = imp.make_version(data, continuum.Operation.UPDATE) + self.assertIsInstance(version, vercls) + self.assertEqual(version.uuid, user.uuid) + self.assertEqual(version.username, "freddie") + self.assertIn(version, self.session) + self.assertIs(version.transaction, imp.continuum_txn) + self.assertEqual(version.operation_type, continuum.Operation.UPDATE) + + def test_create_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + source_data = {"uuid": make_true_uuid(), "username": "bettie"} + self.assertEqual(self.session.query(vercls).count(), 0) + version = imp.create_target_object((source_data["uuid"], 1), source_data) + self.assertEqual(self.session.query(vercls).count(), 1) + self.assertEqual(version.transaction_id, imp.continuum_txn.id) + self.assertEqual(version.operation_type, continuum.Operation.INSERT) + self.assertIsNone(version.end_transaction_id) + + def test_update_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + version1 = user.versions[0] + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + source_data = {"uuid": user.uuid, "username": "freddie"} + target_data = imp.normalize_target_object(version1) + self.assertEqual(self.session.query(vercls).count(), 1) + self.assertIsNone(version1.end_transaction_id) + version2 = imp.update_target_object( + version1, source_data, target_data=target_data + ) + self.assertEqual(self.session.query(vercls).count(), 2) + self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.operation_type, continuum.Operation.UPDATE) + self.assertIsNone(version2.end_transaction_id) + + def test_delete_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + version1 = user.versions[0] + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + self.assertEqual(self.session.query(vercls).count(), 1) + self.assertIsNone(version1.end_transaction_id) + version2 = imp.delete_target_object(version1) + self.assertEqual(self.session.query(vercls).count(), 2) + self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.operation_type, continuum.Operation.DELETE) + self.assertIsNone(version2.end_transaction_id) diff --git a/tests/importing/test_wutta.py b/tests/importing/test_wutta.py index 4d6fdd2..1533605 100644 --- a/tests/importing/test_wutta.py +++ b/tests/importing/test_wutta.py @@ -1,38 +1,3 @@ # -*- coding: utf-8; -*- -from unittest.mock import patch - -from wuttjamaican.testing import DataTestCase - from wuttasync.importing import wutta as mod - - -class TestToWuttaHandler(DataTestCase): - - def make_handler(self, **kwargs): - return mod.ToWuttaHandler(self.config, **kwargs) - - def test_get_target_title(self): - handler = self.make_handler() - - # uses app title by default - self.config.setdefault("wutta.app_title", "What About This") - self.assertEqual(handler.get_target_title(), "What About This") - - # or generic default if present - handler.generic_target_title = "WHATABOUTTHIS" - self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS") - - # but prefer specific title if present - handler.target_title = "what_about_this" - self.assertEqual(handler.get_target_title(), "what_about_this") - - def test_make_target_session(self): - handler = self.make_handler() - - # makes "new" (mocked in our case) app session - with patch.object(self.app, "make_session") as make_session: - make_session.return_value = self.session - session = handler.make_target_session() - make_session.assert_called_once_with() - self.assertIs(session, self.session) From 1e7722de911297f481a0cee51e3bafe96b65a47e Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Thu, 18 Dec 2025 22:57:04 -0600 Subject: [PATCH 2/6] fix: add `--comment` param for `import-versions` command --- src/wuttasync/cli/import_versions.py | 10 ++++++++- src/wuttasync/importing/versions.py | 13 +++++++++++ tests/importing/test_versions.py | 33 ++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/wuttasync/cli/import_versions.py b/src/wuttasync/cli/import_versions.py index f1d0481..86da4c4 100644 --- a/src/wuttasync/cli/import_versions.py +++ b/src/wuttasync/cli/import_versions.py @@ -28,6 +28,7 @@ import sys import rich import typer +from typing_extensions import Annotated from wuttjamaican.cli import wutta_typer @@ -36,7 +37,14 @@ from .base import import_command, ImportCommandHandler @wutta_typer.command() @import_command -def import_versions(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument +def import_versions( # pylint: disable=unused-argument + ctx: typer.Context, + comment: Annotated[ + str, + typer.Option("--comment", "-m", help="Comment to set on the transaction."), + ] = "import catch-up versions", + **kwargs, +): """ Import latest data to version tables, for Wutta DB """ diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py index 53c25fa..b2fd062 100644 --- a/src/wuttasync/importing/versions.py +++ b/src/wuttasync/importing/versions.py @@ -92,6 +92,13 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler): See also :attr:`continuum_uow`. """ + continuum_comment = None + + def consume_kwargs(self, kwargs): + kwargs = super().consume_kwargs(kwargs) + self.continuum_comment = kwargs.pop("comment", None) + return kwargs + def begin_target_transaction(self): # pylint: disable=line-too-long """ @@ -106,6 +113,8 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler): :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()` and assigns that to :attr:`continuum_txn`. + It also sets the comment for the transaction, if applicable. + See also docs for parent method: :meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()` """ @@ -116,8 +125,12 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler): self.continuum_uow = continuum.versioning_manager.unit_of_work( self.target_session ) + self.continuum_txn = self.continuum_uow.create_transaction(self.target_session) + if self.continuum_comment: + self.continuum_txn.meta = {"comment": self.continuum_comment} + def get_importer_kwargs(self, key, **kwargs): """ This modifies the new importer kwargs to add: diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py index 2067f93..1988706 100644 --- a/tests/importing/test_versions.py +++ b/tests/importing/test_versions.py @@ -14,17 +14,40 @@ class TestFromWuttaToVersions(VersionTestCase): def make_handler(self, **kwargs): return mod.FromWuttaToVersions(self.config, **kwargs) + def test_consume_kwargs(self): + + # no comment by default + handler = self.make_handler() + kw = handler.consume_kwargs({}) + self.assertEqual(kw, {}) + self.assertIsNone(handler.continuum_comment) + + # but can provide one + handler = self.make_handler() + kw = handler.consume_kwargs({"comment": "yeehaw"}) + self.assertEqual(kw, {}) + self.assertEqual(handler.continuum_comment, "yeehaw") + def test_begin_target_transaction(self): model = self.app.model txncls = continuum.transaction_class(model.User) + # basic / defaults handler = self.make_handler() self.assertIsNone(handler.continuum_uow) self.assertIsNone(handler.continuum_txn) - handler.begin_target_transaction() self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork) self.assertIsInstance(handler.continuum_txn, txncls) + # nb. no comment + self.assertIsNone(handler.continuum_txn.meta.get("comment")) + + # with comment + handler = self.make_handler() + handler.continuum_comment = "yeehaw" + handler.begin_target_transaction() + self.assertIn("comment", handler.continuum_txn.meta) + self.assertEqual(handler.continuum_txn.meta["comment"], "yeehaw") def test_get_importer_kwargs(self): handler = self.make_handler() @@ -57,14 +80,6 @@ class TestFromWuttaToVersions(VersionTestCase): self.assertNotIn("Upgrade", importers) -class UserImporter(mod.FromWuttaToVersionBase): - - @property - def model_class(self): - model = self.app.model - return model.User - - class TestFromWuttaToVersionBase(VersionTestCase): def make_importer(self, model_class=None, **kwargs): From 19574ea4a0d0ce52fe3550b504b69cb6d0807c3c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Dec 2025 15:32:15 -0600 Subject: [PATCH 3/6] feat: add `warnings` mode for import/export handlers, commands can now specify `--warn` for import/export CLI, to get diff email when changes occur. this also adds `get_import_handler()` and friends, via app provider. also declare email settings for the 2 existing importers --- docs/api/wuttasync.app.rst | 6 + docs/api/wuttasync.emails.rst | 6 + docs/api/wuttasync.testing.rst | 6 + docs/glossary.rst | 20 ++ docs/index.rst | 5 +- pyproject.toml | 8 + src/wuttasync/app.py | 225 ++++++++++++++++++ src/wuttasync/cli/base.py | 26 +- .../import_export_warning.html.mako | 88 +++++++ src/wuttasync/emails.py | 166 +++++++++++++ src/wuttasync/importing/handlers.py | 206 ++++++++++++++-- src/wuttasync/importing/versions.py | 4 +- src/wuttasync/testing.py | 68 ++++++ tests/importing/test_base.py | 5 +- tests/importing/test_handlers.py | 126 +++++++++- tests/importing/test_versions.py | 4 +- tests/test_app.py | 126 ++++++++++ tests/test_emails.py | 81 +++++++ 18 files changed, 1150 insertions(+), 26 deletions(-) create mode 100644 docs/api/wuttasync.app.rst create mode 100644 docs/api/wuttasync.emails.rst create mode 100644 docs/api/wuttasync.testing.rst create mode 100644 src/wuttasync/app.py create mode 100644 src/wuttasync/email-templates/import_export_warning.html.mako create mode 100644 src/wuttasync/emails.py create mode 100644 src/wuttasync/testing.py create mode 100644 tests/test_app.py create mode 100644 tests/test_emails.py diff --git a/docs/api/wuttasync.app.rst b/docs/api/wuttasync.app.rst new file mode 100644 index 0000000..90f3fa7 --- /dev/null +++ b/docs/api/wuttasync.app.rst @@ -0,0 +1,6 @@ + +``wuttasync.app`` +================= + +.. automodule:: wuttasync.app + :members: diff --git a/docs/api/wuttasync.emails.rst b/docs/api/wuttasync.emails.rst new file mode 100644 index 0000000..63bf435 --- /dev/null +++ b/docs/api/wuttasync.emails.rst @@ -0,0 +1,6 @@ + +``wuttasync.emails`` +==================== + +.. automodule:: wuttasync.emails + :members: diff --git a/docs/api/wuttasync.testing.rst b/docs/api/wuttasync.testing.rst new file mode 100644 index 0000000..e6f1877 --- /dev/null +++ b/docs/api/wuttasync.testing.rst @@ -0,0 +1,6 @@ + +``wuttasync.testing`` +===================== + +.. automodule:: wuttasync.testing + :members: diff --git a/docs/glossary.rst b/docs/glossary.rst index c58e3d6..6a28b11 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -6,6 +6,26 @@ Glossary .. glossary:: :sorted: + import/export key + Unique key representing a particular type of import/export job, + i.e. the source/target and orientation (import vs. export). + + For instance "Wutta → CSV export" uses the key: + ``export.to_csv.from_wutta`` + + More than one :term:`import handler` can share a key, e.g. one + may subclass another and inherit the key. + + However only one handler is "designated" for a given key; it will + be used by default for running those jobs. + + This key is used for lookup in + :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`. + + See also + :meth:`~wuttasync.importing.handlers.ImportHandler.get_key()` + method on the import/export handler. + import handler This a type of :term:`handler` which is responsible for a particular set of data import/export task(s). diff --git a/docs/index.rst b/docs/index.rst index 6fe554a..e6fea22 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -67,13 +67,15 @@ cf. :doc:`rattail-manual:data/sync/index`. .. toctree:: :maxdepth: 1 - :caption: API + :caption: Package API api/wuttasync + api/wuttasync.app api/wuttasync.cli api/wuttasync.cli.base api/wuttasync.cli.import_csv api/wuttasync.cli.import_versions + api/wuttasync.emails api/wuttasync.importing api/wuttasync.importing.base api/wuttasync.importing.csv @@ -81,4 +83,5 @@ cf. :doc:`rattail-manual:data/sync/index`. api/wuttasync.importing.model api/wuttasync.importing.versions api/wuttasync.importing.wutta + api/wuttasync.testing api/wuttasync.util diff --git a/pyproject.toml b/pyproject.toml index cff065a..51a1a70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ + "humanize", "makefun", "SQLAlchemy-Utils", "WuttJamaican[db]>=0.16.2", @@ -37,6 +38,13 @@ docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"] +[project.entry-points."wutta.app.providers"] +wuttasync = "wuttasync.app:WuttaSyncAppProvider" + +[project.entry-points."wuttasync.importing"] +"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions" +"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta" + [project.entry-points."wutta.typer_imports"] wuttasync = "wuttasync.cli" diff --git a/src/wuttasync/app.py b/src/wuttasync/app.py new file mode 100644 index 0000000..0fa19fd --- /dev/null +++ b/src/wuttasync/app.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +App handler supplement for WuttaSync +""" + +from collections import OrderedDict + +from wuttjamaican.app import AppProvider +from wuttjamaican.util import load_entry_points + + +class WuttaSyncAppProvider(AppProvider): + """ + The :term:`app provider` for WuttaSync. + + This adds some methods to the :term:`app handler`, which are + specific to import/export. + + It also declares some :term:`email modules ` and + :term:`email templates ` for the app. + + We have two concerns when doing lookups etc. for import/export + handlers: + + * which handlers are *available* - i.e. they exist and are + discoverable + * which handlers are *designated* - only one designated handler + per key + + All "available" handlers will have a key, but some keys may be + referenced by multiple handlers. For each key, only one handler + can be "designated" - there is a default, but config can override. + """ + + email_modules = ["wuttasync.emails"] + email_templates = ["wuttasync:email-templates"] + + def get_all_import_handlers(self): + """ + Returns *all* :term:`import/export handler ` + *classes* which are known to exist, i.e. are discoverable. + + See also :meth:`get_import_handler()` and + :meth:`get_designated_import_handlers()`. + + The discovery process is as follows: + + * load handlers from registered entry points + * check config for designated handlers + + Checking for designated handler config is not a reliable way + to discover handlers, but it's done just in case any new ones + might be found. + + Registration via entry points is the only way to ensure a + handler is discoverable. The entry point group name is always + ``wuttasync.importing`` regardless of :term:`app name`; + entries are like ``"handler_key" = "handler_spec"``. For + example: + + .. code-block:: toml + + [project.entry-points."wuttasync.importing"] + "export.to_csv.from_poser" = "poser.exporting.csv:FromPoserToCsv" + "import.to_poser.from_csv" = "poser.importing.csv:FromCsvToPoser" + + :returns: List of all import/export handler classes + """ + # first load all "registered" Handler classes + factories = load_entry_points("wuttasync.importing", ignore_errors=True) + + # organize registered classes by spec + specs = {factory.get_spec(): factory for factory in factories.values()} + + # many handlers may not be registered per se, but may be + # designated via config. so try to include those too + for factory in factories.values(): + spec = self.get_designated_import_handler_spec(factory.get_key()) + if spec and spec not in specs: + specs[spec] = self.app.load_object(spec) + + # flatten back to simple list of classes + factories = list(specs.values()) + return factories + + def get_designated_import_handler_spec(self, key, require=False): + """ + Returns the designated import/export handler :term:`spec` + string for the given type key. + + This just checks config for the designated handler, using the + ``wuttasync.importing`` prefix regardless of :term:`app name`. + For instance: + + .. code-block:: ini + + [wuttasync.importing] + export.to_csv.from_poser.handler = poser.exporting.csv:FromPoserToCsv + import.to_poser.from_csv.handler = poser.importing.csv:FromCsvToPoser + + See also :meth:`get_designated_import_handlers()` and + :meth:`get_import_handler()`. + + :param key: Unique key indicating the type of import/export + handler. + + :param require: Flag indicating whether an error should be raised if no + handler is found. + + :returns: Spec string for the designated handler. If none is + configured, then ``None`` is returned *unless* the + ``require`` param is true, in which case an error is + raised. + """ + spec = self.config.get(f"wuttasync.importing.{key}.handler") + if spec: + return spec + + spec = self.config.get(f"wuttasync.importing.{key}.default_handler") + if spec: + return spec + + if require: + raise ValueError(f"Cannot locate import handler spec for key: {key}") + return None + + def get_designated_import_handlers(self): + """ + Returns all *designated* import/export handler *instances*. + + Each import/export handler has a "key" which indicates the + "type" of import/export job it performs. For instance the CSV + → Wutta import has the key: ``import.to_wutta.from_csv`` + + More than one handler can be defined for that key; however + only one such handler will be "designated" for each key. + + This method first loads *all* available import handlers, then + organizes them by key, and tries to determine which handler + should be designated for each key. + + See also :meth:`get_all_import_handlers()` and + :meth:`get_designated_import_handler_spec()`. + + :returns: List of designated import/export handler instances + """ + grouped = OrderedDict() + for factory in self.get_all_import_handlers(): + key = factory.get_key() + grouped.setdefault(key, []).append(factory) + + def find_designated(key, group): + spec = self.get_designated_import_handler_spec(key) + if spec: + for factory in group: + if factory.get_spec() == spec: + return factory + if len(group) == 1: + return group[0] + return None + + designated = [] + for key, group in grouped.items(): + factory = find_designated(key, group) + if factory: + handler = factory(self.config) + designated.append(handler) + + return designated + + def get_import_handler(self, key, require=False, **kwargs): + """ + Returns the designated :term:`import/export handler ` instance for the given :term:`import/export key`. + + See also :meth:`get_all_import_handlers()` and + :meth:`get_designated_import_handlers()`. + + :param key: Key indicating the type of import/export handler, + e.g. ``"import.to_wutta.from_csv"`` + + :param require: Set this to true if you want an error raised + when no handler is found. + + :returns: The import/export handler instance. If no handler + is found, then ``None`` is returned, unless ``require`` + param is true, in which case error is raised. + """ + # first try to fetch the handler per designated spec + spec = self.get_designated_import_handler_spec(key, **kwargs) + if spec: + factory = self.app.load_object(spec) + return factory(self.config) + + # nothing was designated, so leverage logic which already + # sorts out which handler is "designated" for given key + designated = self.get_designated_import_handlers() + for handler in designated: + if handler.get_key() == key: + return handler + + if require: + raise ValueError(f"Cannot locate import handler for key: {key}") + return None diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index 08fa4f5..a2460d5 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -127,15 +127,16 @@ class ImportCommandHandler(GenericHandler): This is what happens when command line has ``--list-models``. """ - sys.stdout.write("ALL MODELS:\n") + sys.stdout.write("\nALL MODELS:\n") sys.stdout.write("==============================\n") for key in self.import_handler.importers: sys.stdout.write(key) sys.stdout.write("\n") sys.stdout.write("==============================\n") + sys.stdout.write(f"for {self.import_handler.get_title()}\n\n") -def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments +def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments,too-many-locals models: Annotated[ Optional[List[str]], typer.Argument( @@ -217,6 +218,27 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen help="Max number of *any* target record changes which may occur (per model)." ), ] = None, + warnings: Annotated[ + bool, + typer.Option( + "--warn", + "-W", + help="Expect no changes; warn (email the diff) if any occur.", + ), + ] = False, + warnings_recipients: Annotated[ + str, + typer.Option( + "--recip", help="Override the recipient(s) for diff warning email." + ), + ] = None, + warnings_max_diffs: Annotated[ + int, + typer.Option( + "--max-diffs", + help="Max number of record diffs to show (per model) in warning email.", + ), + ] = 15, dry_run: Annotated[ bool, typer.Option( diff --git a/src/wuttasync/email-templates/import_export_warning.html.mako b/src/wuttasync/email-templates/import_export_warning.html.mako new file mode 100644 index 0000000..9be7770 --- /dev/null +++ b/src/wuttasync/email-templates/import_export_warning.html.mako @@ -0,0 +1,88 @@ +## -*- coding: utf-8; -*- + + +

Diff warning for ${title} (${handler.actioning})

+ +

+ % if dry_run: + DRY RUN + - these changes have not yet happened + % else: + LIVE RUN + - these changes already happened + % endif +

+ +
    + % for model, (created, updated, deleted) in changes.items(): +
  • + ${model} - + ${app.render_quantity(len(created))} created; + ${app.render_quantity(len(updated))} updated; + ${app.render_quantity(len(deleted))} deleted +
    • + % endfor +
+ +

+ COMMAND: +   + ${argv} +

+ +

+ RUNTIME: +   + ${runtime} (${runtime_display}) +

+ + % for model, (created, updated, deleted) in changes.items(): + +
+

+ ${model} - + ${app.render_quantity(len(created))} created; + ${app.render_quantity(len(updated))} updated; + ${app.render_quantity(len(deleted))} deleted +

+ +
+ + % for obj, source_data in created[:max_diffs]: +
${model} created in ${target_title}: ${obj}
+ <% diff = make_diff({}, source_data, nature="create") %> +
+ ${diff.render_html()} +
+ % endfor + % if len(created) > max_diffs: +
${model} - ${app.render_quantity(len(created) - max_diffs)} more records created in ${target_title} - not shown here
+ % endif + + % for obj, source_data, target_data in updated[:max_diffs]: +
${model} updated in ${target_title}: ${obj}
+ <% diff = make_diff(target_data, source_data, nature="update") %> +
+ ${diff.render_html()} +
+ % endfor + % if len(updated) > max_diffs: +
${model} - ${app.render_quantity(len(updated) - max_diffs)} more records updated in ${target_title} - not shown here
+ % endif + + % for obj, target_data in deleted[:max_diffs]: +
${model} deleted in ${target_title}: ${obj}
+ <% diff = make_diff(target_data, {}, nature="delete") %> +
+ ${diff.render_html()} +
+ % endfor + % if len(deleted) > max_diffs: +
${model} - ${app.render_quantity(len(deleted) - max_diffs)} more records deleted in ${target_title} - not shown here
+ % endif + +
+ + % endfor + + diff --git a/src/wuttasync/emails.py b/src/wuttasync/emails.py new file mode 100644 index 0000000..b34112d --- /dev/null +++ b/src/wuttasync/emails.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +:term:`Email Settings ` for WuttaSync +""" + +import datetime +import re +from uuid import UUID + +from wuttjamaican.email import EmailSetting +from wuttjamaican.diffs import Diff + + +class ImportExportWarning(EmailSetting): + """ + Base class for import/export diff warnings; sent when unexpected + changes occur. + + This inherits from :class:`~wuttjamaican.email.EmailSetting`. + """ + + fallback_key = "import_export_warning" + "" # suppress docs + + import_handler_spec = None + import_handler_key = None + + def get_description(self): # pylint: disable=empty-docstring + """ """ + handler = self.get_import_handler() + return f"Diff warning email for {handler.actioning} {handler.get_title()}" + + def get_default_subject(self): # pylint: disable=empty-docstring + """ """ + handler = self.get_import_handler() + return f"Changes for {handler.get_title()}" + + def get_import_handler(self): # pylint: disable=missing-function-docstring + + # prefer explicit spec, if set + if self.import_handler_spec: + return self.app.load_object(self.import_handler_spec)(self.config) + + # next try spec lookup, if key set + if self.import_handler_key: + return self.app.get_import_handler(self.import_handler_key, require=True) + + # or maybe try spec lookup basd on setting class name + class_name = self.__class__.__name__ + if match := re.match( + r"^(?Pimport|export)_to_(?P\S+)_from_(?P\S+)_warning$", + class_name, + ): + key = f"{match['action']}.to_{match['target']}.from_{match['source']}" + return self.app.get_import_handler(key, require=True) + + raise ValueError( + "must set import_handler_spec (or import_handler_key) " + f"for email setting: {class_name}" + ) + + # nb. this is just used for sample data + def make_diff(self, *args, **kwargs): # pylint: disable=missing-function-docstring + return Diff(self.config, *args, **kwargs) + + def sample_data(self): # pylint: disable=empty-docstring + """ """ + model = self.app.model + handler = self.get_import_handler() + + alice = model.User(username="alice") + bob = model.User(username="bob") + charlie = model.User(username="charlie") + + runtime = datetime.timedelta(seconds=30) + return { + "handler": handler, + "title": handler.get_title(), + "source_title": handler.get_source_title(), + "target_title": handler.get_target_title(), + "runtime": runtime, + "runtime_display": "30 seconds", + "dry_run": True, + "argv": [ + "bin/wutta", + "import-foo", + "User", + "--delete", + "--dry-run", + "-W", + ], + "changes": { + "User": ( + [ + ( + alice, + { + "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"), + "username": "alice", + }, + ), + ], + [ + ( + bob, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bob", + }, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bobbie", + }, + ), + ], + [ + ( + charlie, + { + "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"), + "username": "charlie", + }, + ), + ], + ), + }, + "make_diff": self.make_diff, + "max_diffs": 15, + } + + +class import_to_versions_from_wutta_warning( # pylint: disable=invalid-name + ImportExportWarning +): + """ + Diff warning for Wutta → Versions import. + """ + + +class import_to_wutta_from_csv_warning( # pylint: disable=invalid-name + ImportExportWarning +): + """ + Diff warning for CSV → Wutta import. + """ diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index c1f7595..2a0ba71 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -26,10 +26,14 @@ Data Import / Export Handlers import logging import os +import sys from collections import OrderedDict from enum import Enum +import humanize + from wuttjamaican.app import GenericHandler +from wuttjamaican.diffs import Diff log = logging.getLogger(__name__) @@ -44,7 +48,7 @@ class Orientation(Enum): EXPORT = "export" -class ImportHandler(GenericHandler): +class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods """ Base class for all import/export handlers. @@ -121,6 +125,47 @@ class ImportHandler(GenericHandler): :meth:`commit_transaction()`. """ + process_started = None + + warnings = False + """ + Boolean indicating the import/export should run in "warnings" + mode. + + If set, this declares that no changes are expected for the + import/export job. If any changes do occur with this flag set, a + diff warning email is sent within :meth:`process_changes()`. + + See also :attr:`warnings_recipients`, + :attr:`warnings_max_diffs` and :attr:`warnings_email_key`. + """ + + warnings_email_key = None + """ + Explicit :term:`email key` for sending the diff warning email, + *unique to this import/export type*. + + Handlers do not normally set this, so the email key is determined + automatically within :meth:`get_warnings_email_key()`. + + See also :attr:`warnings`. + """ + + warnings_recipients = None + """ + Explicit recipient list for the warning email. If not set, the + recipients are determined automatically via config. + + See also :attr:`warnings`. + """ + + warnings_max_diffs = 15 + """ + Max number of record diffs (per model) to show in the warning email. + + See also :attr:`warnings`. + """ + importers = None """ This should be a dict of all importer/exporter classes available @@ -164,18 +209,21 @@ class ImportHandler(GenericHandler): @classmethod def get_key(cls): """ - Returns the "full key" for the handler. This is a combination - of :attr:`source_key` and :attr:`target_key` and - :attr:`orientation`. + Returns the :term:`import/export key` for the handler. This + is a combination of :attr:`source_key` and :attr:`target_key` + and :attr:`orientation`. - For instance in the case of CSV → Wutta, the full handler key - is ``to_wutta.from_csv.import``. + For instance in the case of Wutta → CSV export, the key is: + ``export.to_csv.from_wutta`` - Note that more than one handler may return the same full key - here; but only one will be configured as the "default" handler - for that key. See also :meth:`get_spec()`. + Note that more than one handler may use the same key; but only + one will be configured as the "designated" handler for that + key, a la + :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`. + + See also :meth:`get_spec()`. """ - return f"to_{cls.target_key}.from_{cls.source_key}.{cls.orientation.value}" + return f"{cls.orientation.value}.to_{cls.target_key}.from_{cls.source_key}" @classmethod def get_spec(cls): @@ -278,11 +326,14 @@ class ImportHandler(GenericHandler): * :meth:`begin_transaction()` * :meth:`get_importer()` * :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter) + * :meth:`process_changes()` * :meth:`rollback_transaction()` * :meth:`commit_transaction()` """ kwargs = self.consume_kwargs(kwargs) + self.process_started = self.app.localtime() self.begin_transaction() + changes = OrderedDict() success = False try: @@ -293,22 +344,31 @@ class ImportHandler(GenericHandler): # invoke importer importer = self.get_importer(key, **kwargs) created, updated, deleted = importer.process_data() + changed = bool(created or updated or deleted) # log what happened msg = "%s: added %d; updated %d; deleted %d %s records" if self.dry_run: msg += " (dry run)" - log.info( + logger = log.warning if changed and self.warnings else log.info + logger( msg, self.get_title(), len(created), len(updated), len(deleted), key ) + # keep track of any changes + if changed: + changes[key] = created, updated, deleted + + # post-processing for all changes + if changes: + self.process_changes(changes) + + success = True + except: log.exception("what should happen here?") # TODO raise - else: - success = True - finally: if not success: log.warning("something failed, so transaction was rolled back") @@ -342,6 +402,17 @@ class ImportHandler(GenericHandler): if "dry_run" in kwargs: self.dry_run = kwargs["dry_run"] + if "warnings" in kwargs: + self.warnings = kwargs.pop("warnings") + + if "warnings_recipients" in kwargs: + self.warnings_recipients = self.config.parse_list( + kwargs.pop("warnings_recipients") + ) + + if "warnings_max_diffs" in kwargs: + self.warnings_max_diffs = kwargs.pop("warnings_max_diffs") + return kwargs def begin_transaction(self): @@ -540,6 +611,113 @@ class ImportHandler(GenericHandler): """ return kwargs + def process_changes(self, changes): + """ + Run post-processing operations on the given changes, if + applicable. + + This method is called by :meth:`process_data()`, if any + changes were made. + + Default logic will send a "diff warning" email to the + configured recipient(s), if :attr:`warnings` mode is enabled. + If it is not enabled, nothing happens. + + :param changes: :class:`~python:collections.OrderedDict` of + changes from the overall import/export job. The structure + is described below. + + Keys for the ``changes`` dict will be model/importer names, + for instance:: + + { + "Sprocket": {...}, + "User": {...}, + } + + Value for each model key is a 3-tuple of ``(created, updated, + deleted)``. Each of those elements is a list:: + + { + "Sprocket": ( + [...], # created + [...], # updated + [...], # deleted + ), + } + + The list elements are always tuples, but the structure + varies:: + + { + "Sprocket": ( + [ # created, 2-tuples + (obj, source_data), + ], + [ # updated, 3-tuples + (obj, source_data, target_data), + ], + [ # deleted, 2-tuples + (obj, target_data), + ], + ), + } + """ + if not self.warnings: + return + + def make_diff(*args, **kwargs): + return Diff(self.config, *args, **kwargs) + + runtime = self.app.localtime() - self.process_started + data = { + "handler": self, + "title": self.get_title(), + "source_title": self.get_source_title(), + "target_title": self.get_target_title(), + "dry_run": self.dry_run, + "argv": sys.argv, + "runtime": runtime, + "runtime_display": humanize.naturaldelta(runtime), + "changes": changes, + "make_diff": make_diff, + "max_diffs": self.warnings_max_diffs, + } + + # maybe override recipients + kw = {} + if self.warnings_recipients: + kw["to"] = self.warnings_recipients + # TODO: should we in fact clear these..? + kw["cc"] = [] + kw["bcc"] = [] + + # send the email + email_key = self.get_warnings_email_key() + self.app.send_email(email_key, data, fallback_key="import_export_warning", **kw) + + log.info("%s: warning email was sent", self.get_title()) + + def get_warnings_email_key(self): + """ + Returns the :term:`email key` to be used for sending the diff + warning email. + + The email key should be unique to this import/export type + (really, the :term:`import/export key`) but not necessarily + unique to one handler. + + If :attr:`warnings_email_key` is set, it will be used as-is. + + Otherwise one is generated from :meth:`get_key()`. + + :returns: Email key for diff warnings + """ + if self.warnings_email_key: + return self.warnings_email_key + + return self.get_key().replace(".", "_") + "_warning" + class FromFileHandler(ImportHandler): """ diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py index b2fd062..cda77c9 100644 --- a/src/wuttasync/importing/versions.py +++ b/src/wuttasync/importing/versions.py @@ -297,7 +297,7 @@ class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type obj, self.model_class ): - data["_version"] = obj + data["_objref"] = obj return data @@ -334,7 +334,7 @@ class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): # when we "update" it always involves making a *new* version # record. but that requires actually updating the "previous" # version to indicate the new version's transaction. - prev_version = target_data.pop("_version") + prev_version = target_data.pop("_objref") prev_version.end_transaction_id = self.continuum_txn.id return self.make_version(source_data, continuum.Operation.UPDATE) diff --git a/src/wuttasync/testing.py b/src/wuttasync/testing.py new file mode 100644 index 0000000..1daad1f --- /dev/null +++ b/src/wuttasync/testing.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Testing utilities +""" + +from wuttjamaican.testing import ConfigTestCase + + +class ImportExportWarningTestCase(ConfigTestCase): + """ + Base class for testing the import/export warning email settings. + + This inherits from + :class:`~wuttjamaican:wuttjamaican.testing.ConfigTestCase`. + + Example usage:: + + from wuttasync.testing import ImportExportWarningTestCase + + class TestEmailSettings(ImportExportWarningTestCase): + + def test_import_to_wutta_from_foo_warning(self): + self.do_test_preview("import_to_wutta_from_foo_warning") + + def test_export_to_foo_from_wutta_warning(self): + self.do_test_preview("export_to_foo_from_wutta_warning") + """ + + app_title = "Wutta Poser" + + def setUp(self): + self.setup_config() + self.config.setdefault("wutta.app_title", self.app_title) + + def make_preview( # pylint: disable=missing-function-docstring,unused-argument + self, key, mode="html" + ): + handler = self.app.get_email_handler() + setting = handler.get_email_setting(key) + context = setting.sample_data() + return handler.get_auto_html_body( + setting.key, context, fallback_key=setting.fallback_key + ) + + def do_test_preview(self, key): # pylint: disable=missing-function-docstring + body = self.make_preview(key, mode="html") + self.assertIn("Diff warning for ", body) diff --git a/tests/importing/test_base.py b/tests/importing/test_base.py index c920ed6..9f83ae3 100644 --- a/tests/importing/test_base.py +++ b/tests/importing/test_base.py @@ -717,8 +717,9 @@ class TestFromSqlalchemy(DataTestCase): ) query = imp.get_source_query() self.assertIsInstance(query, orm.Query) - self.assertEqual(len(query.selectable.froms), 1) - table = query.selectable.froms[0] + froms = query.selectable.get_final_froms() + self.assertEqual(len(froms), 1) + table = froms[0] self.assertEqual(table.name, "upgrade") def test_get_source_objects(self): diff --git a/tests/importing/test_handlers.py b/tests/importing/test_handlers.py index a6df032..c01b405 100644 --- a/tests/importing/test_handlers.py +++ b/tests/importing/test_handlers.py @@ -2,12 +2,18 @@ from collections import OrderedDict from unittest.mock import patch +from uuid import UUID from wuttjamaican.testing import DataTestCase from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy +class FromFooToBar(mod.ImportHandler): + source_key = "foo" + target_key = "bar" + + class TestImportHandler(DataTestCase): def make_handler(self, **kwargs): @@ -30,10 +36,10 @@ class TestImportHandler(DataTestCase): def test_get_key(self): handler = self.make_handler() - self.assertEqual(handler.get_key(), "to_None.from_None.import") + self.assertEqual(handler.get_key(), "import.to_None.from_None") with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"): - self.assertEqual(handler.get_key(), "to_wutta.from_csv.import") + self.assertEqual(handler.get_key(), "import.to_wutta.from_csv") def test_get_spec(self): handler = self.make_handler() @@ -149,15 +155,41 @@ class TestImportHandler(DataTestCase): kw = {} result = handler.consume_kwargs(kw) self.assertIs(result, kw) + self.assertEqual(result, {}) - # captures dry-run flag + # dry_run (not consumed) self.assertFalse(handler.dry_run) kw["dry_run"] = True result = handler.consume_kwargs(kw) self.assertIs(result, kw) + self.assertIn("dry_run", kw) self.assertTrue(kw["dry_run"]) self.assertTrue(handler.dry_run) + # warnings (consumed) + self.assertFalse(handler.warnings) + kw["warnings"] = True + result = handler.consume_kwargs(kw) + self.assertIs(result, kw) + self.assertNotIn("warnings", kw) + self.assertTrue(handler.warnings) + + # warnings_recipients (consumed) + self.assertIsNone(handler.warnings_recipients) + kw["warnings_recipients"] = "bob@example.com" + result = handler.consume_kwargs(kw) + self.assertIs(result, kw) + self.assertNotIn("warnings_recipients", kw) + self.assertEqual(handler.warnings_recipients, ["bob@example.com"]) + + # warnings_max_diffs (consumed) + self.assertEqual(handler.warnings_max_diffs, 15) + kw["warnings_max_diffs"] = 30 + result = handler.consume_kwargs(kw) + self.assertIs(result, kw) + self.assertNotIn("warnings_max_diffs", kw) + self.assertEqual(handler.warnings_max_diffs, 30) + def test_define_importers(self): handler = self.make_handler() importers = handler.define_importers() @@ -187,6 +219,94 @@ class TestImportHandler(DataTestCase): KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting ) + def test_get_warnings_email_key(self): + handler = FromFooToBar(self.config) + + # default + key = handler.get_warnings_email_key() + self.assertEqual(key, "import_to_bar_from_foo_warning") + + # override + handler.warnings_email_key = "from_foo_to_bar" + key = handler.get_warnings_email_key() + self.assertEqual(key, "from_foo_to_bar") + + def test_process_changes(self): + model = self.app.model + handler = self.make_handler() + email_handler = self.app.get_email_handler() + + handler.process_started = self.app.localtime() + + alice = model.User(username="alice") + bob = model.User(username="bob") + charlie = model.User(username="charlie") + + changes = { + "User": ( + [ + ( + alice, + { + "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"), + "username": "alice", + }, + ), + ], + [ + ( + bob, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bob", + }, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bobbie", + }, + ), + ], + [ + ( + charlie, + { + "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"), + "username": "charlie", + }, + ), + ], + ), + } + + # no email if not in warnings mode + self.assertFalse(handler.warnings) + with patch.object(self.app, "send_email") as send_email: + handler.process_changes(changes) + send_email.assert_not_called() + + # email sent (to default recip) if in warnings mode + handler.warnings = True + self.config.setdefault("wutta.email.default.to", "admin@example.com") + with patch.object(email_handler, "deliver_message") as deliver_message: + handler.process_changes(changes) + deliver_message.assert_called_once() + args, kwargs = deliver_message.call_args + self.assertEqual(kwargs, {"recips": None}) + self.assertEqual(len(args), 1) + msg = args[0] + self.assertEqual(msg.to, ["admin@example.com"]) + + # can override email recip + handler.warnings_recipients = ["bob@example.com"] + with patch.object(email_handler, "deliver_message") as deliver_message: + handler.process_changes(changes) + deliver_message.assert_called_once() + args, kwargs = deliver_message.call_args + self.assertEqual(kwargs, {"recips": None}) + self.assertEqual(len(args), 1) + msg = args[0] + self.assertEqual(msg.to, ["bob@example.com"]) + class TestFromFileHandler(DataTestCase): diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py index 1988706..2cd4ec0 100644 --- a/tests/importing/test_versions.py +++ b/tests/importing/test_versions.py @@ -132,8 +132,8 @@ class TestFromWuttaToVersionBase(VersionTestCase): # version object should be embedded in data dict data = imp.normalize_target_object(version) self.assertIsInstance(data, dict) - self.assertIn("_version", data) - self.assertIs(data["_version"], version) + self.assertIn("_objref", data) + self.assertIs(data["_objref"], version) # but normal object is not embedded data = imp.normalize_target_object(user) diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..ea2b5e6 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8; -*- + +from wuttjamaican.testing import ConfigTestCase + +from wuttasync import app as mod +from wuttasync.importing import ImportHandler +from wuttasync.importing.csv import FromCsvToWutta + + +class FromFooToBar(ImportHandler): + source_key = "foo" + target_key = "bar" + + +class FromCsvToPoser(FromCsvToWutta): + pass + + +class TestWuttaSyncAppProvider(ConfigTestCase): + + def test_get_all_import_handlers(self): + + # by default our custom handler is not found + handlers = self.app.get_all_import_handlers() + self.assertIn(FromCsvToWutta, handlers) + self.assertNotIn(FromFooToBar, handlers) + + # make sure if we configure a custom handler, it is found + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_csv.handler", + "tests.test_app:FromFooToBar", + ) + handlers = self.app.get_all_import_handlers() + self.assertIn(FromCsvToWutta, handlers) + self.assertIn(FromFooToBar, handlers) + + def test_get_designated_import_handler_spec(self): + + # fetch of unknown key returns none + spec = self.app.get_designated_import_handler_spec("test01") + self.assertIsNone(spec) + + # unless we require it, in which case, error + self.assertRaises( + ValueError, + self.app.get_designated_import_handler_spec, + "test01", + require=True, + ) + + # we configure one for whatever key we like + self.config.setdefault( + "wuttasync.importing.test02.handler", "tests.test_app:FromBarToFoo" + ) + spec = self.app.get_designated_import_handler_spec("test02") + self.assertEqual(spec, "tests.test_app:FromBarToFoo") + + # we can also define a "default" designated handler + self.config.setdefault( + "wuttasync.importing.test03.default_handler", + "tests.test_app:FromBarToFoo", + ) + spec = self.app.get_designated_import_handler_spec("test03") + self.assertEqual(spec, "tests.test_app:FromBarToFoo") + + def test_get_designated_import_handlers(self): + + # some designated handlers exist, but not our custom handler + handlers = self.app.get_designated_import_handlers() + csv_handlers = [ + h for h in handlers if h.get_key() == "import.to_wutta.from_csv" + ] + self.assertEqual(len(csv_handlers), 1) + csv_handler = csv_handlers[0] + self.assertIsInstance(csv_handler, FromCsvToWutta) + self.assertFalse(isinstance(csv_handler, FromCsvToPoser)) + self.assertFalse( + any([h.get_key() == "import.to_bar.from_foo" for h in handlers]) + ) + self.assertFalse(any([isinstance(h, FromFooToBar) for h in handlers])) + self.assertFalse(any([isinstance(h, FromCsvToPoser) for h in handlers])) + self.assertTrue( + any([h.get_key() == "import.to_versions.from_wutta" for h in handlers]) + ) + + # but we can make custom designated + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_csv.handler", + "tests.test_app:FromCsvToPoser", + ) + handlers = self.app.get_designated_import_handlers() + csv_handlers = [ + h for h in handlers if h.get_key() == "import.to_wutta.from_csv" + ] + self.assertEqual(len(csv_handlers), 1) + csv_handler = csv_handlers[0] + self.assertIsInstance(csv_handler, FromCsvToWutta) + self.assertIsInstance(csv_handler, FromCsvToPoser) + self.assertTrue( + any([h.get_key() == "import.to_versions.from_wutta" for h in handlers]) + ) + + def test_get_import_handler(self): + + # make sure a basic fetch works + handler = self.app.get_import_handler("import.to_wutta.from_csv") + self.assertIsInstance(handler, FromCsvToWutta) + self.assertFalse(isinstance(handler, FromCsvToPoser)) + + # and make sure custom override works + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_csv.handler", + "tests.test_app:FromCsvToPoser", + ) + handler = self.app.get_import_handler("import.to_wutta.from_csv") + self.assertIsInstance(handler, FromCsvToWutta) + self.assertIsInstance(handler, FromCsvToPoser) + + # unknown importer cannot be found + handler = self.app.get_import_handler("bogus") + self.assertIsNone(handler) + + # and if we require it, error will raise + self.assertRaises( + ValueError, self.app.get_import_handler, "bogus", require=True + ) diff --git a/tests/test_emails.py b/tests/test_emails.py new file mode 100644 index 0000000..9494753 --- /dev/null +++ b/tests/test_emails.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8; -*- + +from wuttjamaican.testing import ConfigTestCase + +from wuttasync import emails as mod +from wuttasync.importing import ImportHandler +from wuttasync.testing import ImportExportWarningTestCase + + +class FromFooToWutta(ImportHandler): + pass + + +class TestImportExportWarning(ConfigTestCase): + + def make_setting(self, factory=None): + if not factory: + factory = mod.ImportExportWarning + setting = factory(self.config) + return setting + + def test_get_description(self): + self.config.setdefault("wutta.app_title", "Wutta Poser") + setting = self.make_setting() + setting.import_handler_key = "import.to_wutta.from_csv" + self.assertEqual( + setting.get_description(), + "Diff warning email for importing CSV → Wutta Poser", + ) + + def test_get_default_subject(self): + self.config.setdefault("wutta.app_title", "Wutta Poser") + setting = self.make_setting() + setting.import_handler_key = "import.to_wutta.from_csv" + self.assertEqual(setting.get_default_subject(), "Changes for CSV → Wutta Poser") + + def test_get_import_handler(self): + + # nb. typical name pattern + class import_to_wutta_from_foo_warning(mod.ImportExportWarning): + pass + + # nb. name does not match spec pattern + class import_to_wutta_from_bar_blah(mod.ImportExportWarning): + pass + + # register our import handler + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_foo.handler", + "tests.test_emails:FromFooToWutta", + ) + + # error if spec/key not discoverable + setting = self.make_setting(import_to_wutta_from_bar_blah) + self.assertRaises(ValueError, setting.get_import_handler) + + # can lookup by name (auto-spec) + setting = self.make_setting(import_to_wutta_from_foo_warning) + handler = setting.get_import_handler() + self.assertIsInstance(handler, FromFooToWutta) + + # can lookup by explicit spec + setting = self.make_setting(import_to_wutta_from_bar_blah) + setting.import_handler_spec = "tests.test_emails:FromFooToWutta" + handler = setting.get_import_handler() + self.assertIsInstance(handler, FromFooToWutta) + + # can lookup by explicit key + setting = self.make_setting(import_to_wutta_from_bar_blah) + setting.import_handler_key = "import.to_wutta.from_foo" + handler = setting.get_import_handler() + self.assertIsInstance(handler, FromFooToWutta) + + +class TestEmailSettings(ImportExportWarningTestCase): + + def test_import_to_versions_from_wutta_warning(self): + self.do_test_preview("import_to_versions_from_wutta_warning") + + def test_import_to_wutta_from_csv_warning(self): + self.do_test_preview("import_to_wutta_from_csv_warning") From 7e3e8920026bac455cbc836d797ed3d59e6b0c2c Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Dec 2025 16:33:06 -0600 Subject: [PATCH 4/6] fix: allow passing just `key` to ImportCommandHandler so config can override designated handler, and command still work --- src/wuttasync/cli/base.py | 31 ++++++++++++++++++++++------ src/wuttasync/cli/import_csv.py | 4 +--- src/wuttasync/cli/import_versions.py | 4 +--- tests/cli/test_base.py | 4 ++++ 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index a2460d5..e972e18 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -50,19 +50,35 @@ class ImportCommandHandler(GenericHandler): create this handler and call its :meth:`run()` method. This handler does not know how to import/export data, but it knows - how to make its :attr:`import_handler` do it. + how to make its :attr:`import_handler` do it. Likewise, the + import handler is not "CLI-aware" - so this provides the glue. :param import_handler: During construction, caller can specify the :attr:`import_handler` as any of: * import handler instance * import handler factory (e.g. class) - * import handler spec (cf. :func:`~wuttjamaican:wuttjamaican.util.load_object()`) + * import handler :term:`spec` - For example:: + :param key: Optional :term:`import/export key` to use for handler + lookup. Only used if ``import_handler`` param is not set. - handler = ImportCommandHandler( - config, import_handler='wuttasync.importing.csv:FromCsvToWutta') + Typical usage for custom commands will be to provide the spec:: + + handler = ImportCommandHandler( + config, "poser.importing.foo:FromFooToPoser" + ) + + Library authors may prefer to use the import/export key; this lets + the command work with any designated handler:: + + handler = ImportCommandHandler( + config, key="import.to_poser.from_foo" + ) + + See also + :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()` + which does the lookup by key. """ import_handler = None @@ -71,7 +87,7 @@ class ImportCommandHandler(GenericHandler): invoked when command runs. See also :meth:`run()`. """ - def __init__(self, config, import_handler=None): + def __init__(self, config, import_handler=None, key=None): super().__init__(config) if import_handler: @@ -83,6 +99,9 @@ class ImportCommandHandler(GenericHandler): factory = self.app.load_object(import_handler) self.import_handler = factory(self.config) + elif key: + self.import_handler = self.app.get_import_handler(key, require=True) + def run(self, params, progress=None): # pylint: disable=unused-argument """ Run the import/export job(s) based on command line params. diff --git a/src/wuttasync/cli/import_csv.py b/src/wuttasync/cli/import_csv.py index d3c8047..4c5694a 100644 --- a/src/wuttasync/cli/import_csv.py +++ b/src/wuttasync/cli/import_csv.py @@ -38,7 +38,5 @@ def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument Import data from CSV file(s) to Wutta DB """ config = ctx.parent.wutta_config - handler = ImportCommandHandler( - config, import_handler="wuttasync.importing.csv:FromCsvToWutta" - ) + handler = ImportCommandHandler(config, key="import.to_wutta.from_csv") handler.run(ctx.params) diff --git a/src/wuttasync/cli/import_versions.py b/src/wuttasync/cli/import_versions.py index 86da4c4..aa82088 100644 --- a/src/wuttasync/cli/import_versions.py +++ b/src/wuttasync/cli/import_versions.py @@ -69,7 +69,5 @@ def import_versions( # pylint: disable=unused-argument ) sys.exit(1) - handler = ImportCommandHandler( - config, import_handler="wuttasync.importing.versions:FromWuttaToVersions" - ) + handler = ImportCommandHandler(config, key="import.to_versions.from_wutta") handler.run(ctx.params) diff --git a/tests/cli/test_base.py b/tests/cli/test_base.py index 991358e..b8fc954 100644 --- a/tests/cli/test_base.py +++ b/tests/cli/test_base.py @@ -34,6 +34,10 @@ class TestImportCommandHandler(DataTestCase): handler = self.make_handler(import_handler=myhandler) self.assertIs(handler.import_handler, myhandler) + # as key + handler = self.make_handler(key="import.to_wutta.from_csv") + self.assertIsInstance(handler.import_handler, FromCsvToWutta) + def test_run(self): handler = self.make_handler( import_handler="wuttasync.importing.csv:FromCsvToWutta" From 4f800852549162dedfd66265bc248e2e6d8038bd Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Dec 2025 17:28:06 -0600 Subject: [PATCH 5/6] fix: run all models when none specified, for import/export commands --- src/wuttasync/cli/base.py | 12 ++++++------ src/wuttasync/importing/base.py | 8 +++++--- src/wuttasync/importing/handlers.py | 3 +++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index e972e18..56d6421 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -125,15 +125,15 @@ class ImportCommandHandler(GenericHandler): return # otherwise process some data + log.debug("using handler: %s", self.import_handler.get_spec()) kw = dict(params) models = kw.pop("models") - log.debug("using handler: %s", self.import_handler.get_spec()) - # TODO: need to use all/default models if none specified - # (and should know models by now for logging purposes) + if not models: + models = list(self.import_handler.importers) log.debug( - "running %s %s for: %s", - self.import_handler, - self.import_handler.orientation.value, + "%s %s for models: %s", + self.import_handler.actioning, + self.import_handler.get_title(), ", ".join(models), ) log.debug("params are: %s", kw) diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py index ca0718e..b625b17 100644 --- a/src/wuttasync/importing/base.py +++ b/src/wuttasync/importing/base.py @@ -459,7 +459,10 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- updated = [] deleted = [] - log.debug("using key fields: %s", ", ".join(self.get_keys())) + model_title = self.get_model_title() + log.debug( + "using key fields for %s: %s", model_title, ", ".join(self.get_keys()) + ) # get complete set of normalized source data if source_data is None: @@ -468,8 +471,7 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- # nb. prune duplicate records from source data source_data, source_keys = self.get_unique_data(source_data) - model_title = self.get_model_title() - log.debug(f"got %s {model_title} records from source", len(source_data)) + log.debug("got %s %s records from source", len(source_data), model_title) # maybe cache existing target data if self.caches_target: diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index 2a0ba71..ac13f28 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -335,6 +335,9 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods self.begin_transaction() changes = OrderedDict() + if not keys: + keys = list(self.importers) + success = False try: From 8c5918b9fb9572b096c60e989aa562b361f8c3b5 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sat, 20 Dec 2025 20:24:57 -0600 Subject: [PATCH 6/6] =?UTF-8?q?bump:=20version=200.2.1=20=E2=86=92=200.3.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 19 +++++++++++++++++++ pyproject.toml | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1fd1a..ef2f351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ All notable changes to WuttaSync will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## v0.3.0 (2025-12-20) + +### Feat + +- add `warnings` mode for import/export handlers, commands +- add the `import-versions` command, handler logic + +### Fix + +- run all models when none specified, for import/export commands +- allow passing just `key` to ImportCommandHandler +- add `--comment` param for `import-versions` command +- add basic data type coercion for CSV -> SQLAlchemy import +- refactor some more for tests + pylint +- refactor per pylint; add to tox +- format all code with black +- tweak logging when deleting object +- add logging when deleting target object + ## v0.2.1 (2025-06-29) ### Fix diff --git a/pyproject.toml b/pyproject.toml index 51a1a70..74ab85a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "WuttaSync" -version = "0.2.1" +version = "0.3.0" description = "Wutta Framework for data import/export and real-time sync" readme = "README.md" authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}] @@ -29,13 +29,13 @@ dependencies = [ "humanize", "makefun", "SQLAlchemy-Utils", - "WuttJamaican[db]>=0.16.2", + "WuttJamaican[db]>=0.27.0", ] [project.optional-dependencies] docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] -tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"] +tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"] [project.entry-points."wutta.app.providers"]