diff --git a/docs/api/wuttasync.cli.import_versions.rst b/docs/api/wuttasync.cli.import_versions.rst new file mode 100644 index 0000000..aeb8227 --- /dev/null +++ b/docs/api/wuttasync.cli.import_versions.rst @@ -0,0 +1,6 @@ + +``wuttasync.cli.import_versions`` +================================= + +.. automodule:: wuttasync.cli.import_versions + :members: diff --git a/docs/api/wuttasync.importing.versions.rst b/docs/api/wuttasync.importing.versions.rst new file mode 100644 index 0000000..aa970a1 --- /dev/null +++ b/docs/api/wuttasync.importing.versions.rst @@ -0,0 +1,6 @@ + +``wuttasync.importing.versions`` +================================ + +.. automodule:: wuttasync.importing.versions + :members: diff --git a/docs/conf.py b/docs/conf.py index 2b47550..7826856 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,13 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None), + "sqlalchemy": ("http://docs.sqlalchemy.org/en/latest/", None), + "sqlalchemy-continuum": ( + "https://sqlalchemy-continuum.readthedocs.io/en/latest/", + None, + ), + "sqlalchemy-utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None), + "wutta-continuum": ("https://docs.wuttaproject.org/wutta-continuum/", None), "wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None), } diff --git a/docs/index.rst b/docs/index.rst index 9eb2d93..6fe554a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,10 +73,12 @@ cf. :doc:`rattail-manual:data/sync/index`. api/wuttasync.cli api/wuttasync.cli.base api/wuttasync.cli.import_csv + api/wuttasync.cli.import_versions api/wuttasync.importing api/wuttasync.importing.base api/wuttasync.importing.csv api/wuttasync.importing.handlers api/wuttasync.importing.model + api/wuttasync.importing.versions api/wuttasync.importing.wutta api/wuttasync.util diff --git a/docs/narr/cli/builtin.rst b/docs/narr/cli/builtin.rst index 0630c94..ac6fb14 100644 --- a/docs/narr/cli/builtin.rst +++ b/docs/narr/cli/builtin.rst @@ -25,3 +25,24 @@ types may not behave as expected etc. Defined in: :mod:`wuttasync.cli.import_csv` .. program-output:: wutta import-csv --help + + +.. _wutta-import-versions: + +``wutta import-versions`` +------------------------- + +Import latest data to version tables, for the Wutta :term:`app +database`. + +The purpose of this is to ensure version tables accurately reflect +the current "live" data set, for given table(s). It is only +relevant/usable if versioning is configured and enabled. For more +on that see :doc:`wutta-continuum:index`. + +This command can check/update version tables for any versioned class +in the :term:`app model`. + +Defined in: :mod:`wuttasync.cli.import_versions` + +.. program-output:: wutta import-versions --help diff --git a/pyproject.toml b/pyproject.toml index a48b949..cff065a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] -tests = ["pylint", "pytest", "pytest-cov", "tox"] +tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"] [project.entry-points."wutta.typer_imports"] diff --git a/src/wuttasync/cli/__init__.py b/src/wuttasync/cli/__init__.py index c77a4e2..0d88ed4 100644 --- a/src/wuttasync/cli/__init__.py +++ b/src/wuttasync/cli/__init__.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -34,3 +34,4 @@ from .base import import_command, file_import_command, ImportCommandHandler # nb. must bring in all modules for discovery to work from . import import_csv +from . import import_versions diff --git a/src/wuttasync/cli/import_versions.py b/src/wuttasync/cli/import_versions.py new file mode 100644 index 0000000..f1d0481 --- /dev/null +++ b/src/wuttasync/cli/import_versions.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +See also: :ref:`wutta-import-versions` +""" + +import sys + +import rich +import typer + +from wuttjamaican.cli import wutta_typer + +from .base import import_command, ImportCommandHandler + + +@wutta_typer.command() +@import_command +def import_versions(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument + """ + Import latest data to version tables, for Wutta DB + """ + config = ctx.parent.wutta_config + app = config.get_app() + + # warn/exit if libs are not installed + try: + import wutta_continuum # pylint: disable=import-outside-toplevel,unused-import + except ImportError: # pragma: no cover + rich.print( + "\n\t[bold yellow]Wutta-Continum is not installed![/bold yellow]\n" + "\n\tIf you want it, run: pip install Wutta-Continuum\n" + ) + sys.exit(1) + + # warn/exit if feature disabled + if not app.continuum_is_enabled(): # pragma: no cover + rich.print( + "\n\t[bold yellow]Wutta-Continum is not enabled![/bold yellow]\n" + "\n\tIf you want it, see: https://docs.wuttaproject.org/wutta-continuum/\n" + ) + sys.exit(1) + + handler = ImportCommandHandler( + config, import_handler="wuttasync.importing.versions:FromWuttaToVersions" + ) + handler.run(ctx.params) diff --git a/src/wuttasync/importing/__init__.py b/src/wuttasync/importing/__init__.py index 03a421f..545cbb9 100644 --- a/src/wuttasync/importing/__init__.py +++ b/src/wuttasync/importing/__init__.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -32,7 +32,7 @@ And some :term:`import handler` base classes: * :class:`~wuttasync.importing.handlers.ImportHandler` * :class:`~wuttasync.importing.handlers.FromFileHandler` * :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler` -* :class:`~wuttasync.importing.wutta.ToWuttaHandler` +* :class:`~wuttasync.importing.handlers.ToWuttaHandler` And some :term:`importer` base classes: @@ -42,7 +42,12 @@ And some :term:`importer` base classes: * :class:`~wuttasync.importing.model.ToWutta` """ -from .handlers import Orientation, ImportHandler, FromFileHandler, ToSqlalchemyHandler +from .handlers import ( + Orientation, + ImportHandler, + FromFileHandler, + ToSqlalchemyHandler, + ToWuttaHandler, +) from .base import Importer, FromFile, ToSqlalchemy from .model import ToWutta -from .wutta import ToWuttaHandler diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py index 629ead6..ca0718e 100644 --- a/src/wuttasync/importing/base.py +++ b/src/wuttasync/importing/base.py @@ -184,6 +184,19 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- :meth:`get_target_cache()`. """ + default_keys = None + """ + In certain edge cases, the importer class must declare its key + list without using :attr:`keys`. + + (As of now this only happens with + :class:`~wuttasync.importing.versions.FromWuttaToVersions` which + must dynamically create importer classes.) + + If applicable, this value is used as fallback for + :meth:`get_keys()`. + """ + max_create = None max_update = None max_delete = None @@ -323,19 +336,54 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- def get_keys(self): """ - Must return the key field(s) for use with import/export. + Retrieve the list of key field(s) for use with import/export. + The result is cached, so the key list is only calculated once. + + Many importers have just one key field, but we always assume a + key *list* - so this often is a list with just one field. All fields in this list should also be found in the output for :meth:`get_fields()`. + Many importers will declare this via :attr:`keys` (or + :attr:`key`) static attribute:: + + class SprocketImporter(Importer): + + # nb. all these examples work the same + + # 'keys' is the preferred attribute + keys = ("sprocket_id",) # <-- the "canonical" way + keys = ["sprocket_id"] + keys = "sprocket_id" + + # 'key' is not preferred, but works + key = ("sprocket_id",) + key = "sprocket_id" + + If neither ``keys`` nor ``key`` is set, as a special case + :attr:`default_keys` is used if set. + + If no keys were declared, the list is inspected from the model + class via + :func:`sqlalchemy-utils:sqlalchemy_utils.functions.get_primary_keys()`. + + In any case, the determination is made only once. This method + also *sets* :attr:`keys` on the instance, so it will return + that as-is for subsequent calls. + :returns: List of "key" field names. """ keys = None + # nb. prefer 'keys' but use 'key' as fallback if "keys" in self.__dict__: keys = self.__dict__["keys"] elif "key" in self.__dict__: keys = self.__dict__["key"] + else: + keys = self.default_keys + if keys: if isinstance(keys, str): keys = self.config.parse_list(keys) @@ -1271,10 +1319,139 @@ class FromFile(Importer): self.input_file.close() +class QueryWrapper: + """ + Simple wrapper for a SQLAlchemy query, to make it sort of behave + so that an importer can treat it as a data record list. + + :param query: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance + """ + + def __init__(self, query): + self.query = query + + def __len__(self): + try: + return len(self.query) + except TypeError: + return self.query.count() + + def __iter__(self): + return iter(self.query) + + +class FromSqlalchemy(Importer): # pylint: disable=abstract-method + """ + Base class for importer/exporter using SQL/ORM query as data + source. + + Subclass should define :attr:`source_model_class` in which case + the source query is automatic. And/or override + :meth:`get_source_query()` to customize. + + See also :class:`FromSqlalchemyMirror` and :class:`ToSqlalchemy`. + """ + + source_model_class = None + """ + Reference to the :term:`data model` class representing the source. + + This normally is a SQLAlchemy mapped class, e.g. + :class:`~wuttjamaican:wuttjamaican.db.model.base.Person` for + exporting from the Wutta People table. + """ + + source_session = None + """ + Reference to the open :term:`db session` for the data source. + + The importer must be given this reference when instantiated by the + :term:`import handler`. This is handled automatically if using + :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`. + """ + + def get_source_objects(self): + """ + This method is responsible for fetching "raw" (non-normalized) + records from data source. + + (See also the parent method docs for + :meth:`~wuttasync.importing.base.Importer.get_source_objects()`.) + + It calls :meth:`get_source_query()` and then wraps that in a + :class:`QueryWrapper`, which is then returned. + + Note that this method does not technically "retrieve" records + from the query; that happens automatically later. + + :returns: :class:`QueryWrapper` for the source query + """ + query = self.get_source_query() + return QueryWrapper(query) + + def get_source_query(self): + """ + This returns the SQL/ORM query used to fetch source + data. It is called from :meth:`get_source_objects()`. + + Default logic just makes a simple ``SELECT * FROM TABLE`` kind + of query. Subclass can override as needed. + + :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance + """ + return self.source_session.query(self.source_model_class) + + +class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method + """ + Special base class for when the source and target are effectively + mirrored, and can each be represented by the same :term:`data + model`. + + The assumption is that SQLAlchemy ORM is used on both sides, even + though this base class only defines the source side (it inherits + from :class:`FromSqlalchemy`). + + There are two main use cases for this: + + * sync between app nodes + * sync version tables + + When 2 app nodes are synced, the source and target are "the same" + in a schema sense, e.g. ``sprockets on node 01 => sprockets on + node 02``. + + When version tables are synced, the same schema can be used for + the "live" table and the "version" table, e.g. ``sprockets => + sprocket versions``. + """ + + @property + def source_model_class(self): + """ + This returns the :attr:`~Importer.model_class` since source + and target must share common schema. + """ + return self.model_class + + def normalize_source_object(self, obj): + """ + Since source/target share schema, there should be no tricky + normalization involved. + + This calls :meth:`~Importer.normalize_target_object()` since + that logic should already be defined. This ensures the same + normalization is used on both sides. + """ + return self.normalize_target_object(obj) + + class ToSqlalchemy(Importer): """ Base class for importer/exporter which uses SQLAlchemy ORM on the target side. + + See also :class:`FromSqlalchemy`. """ caches_target = True @@ -1312,6 +1489,8 @@ class ToSqlalchemy(Importer): Returns an ORM query suitable to fetch existing objects from the target side. This is called from :meth:`get_target_objects()`. + + :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance """ return self.target_session.query(self.model_class) diff --git a/src/wuttasync/importing/csv.py b/src/wuttasync/importing/csv.py index ab0bf21..9190099 100644 --- a/src/wuttasync/importing/csv.py +++ b/src/wuttasync/importing/csv.py @@ -38,8 +38,7 @@ from wuttjamaican.db.util import make_topo_sortkey, UUID from wuttjamaican.util import parse_bool from .base import FromFile -from .handlers import FromFileHandler -from .wutta import ToWuttaHandler +from .handlers import FromFileHandler, ToWuttaHandler from .model import ToWutta @@ -239,6 +238,8 @@ class FromCsvToSqlalchemyHandlerMixin: """ raise NotImplementedError + # TODO: pylint (correctly) flags this as duplicate code, matching + # on the wuttasync.importing.versions module - should fix? def define_importers(self): """ This mixin overrides typical (manual) importer definition, and @@ -252,6 +253,7 @@ class FromCsvToSqlalchemyHandlerMixin: importers = {} model = self.get_target_model() + # pylint: disable=duplicate-code # mostly try to make an importer for every data model for name in dir(model): cls = getattr(model, name) diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index e9c6ac3..c1f7595 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -209,6 +209,20 @@ class ImportHandler(GenericHandler): """ Returns the display title for the data source. + By default this returns :attr:`source_key`, but this can be + overriden by class attribute. + + Base class can define ``generic_source_title`` to provide a + new default:: + + class FromExcelHandler(ImportHandler): + generic_source_title = "Excel File" + + Subclass can define ``source_title`` to be explicit:: + + class FromExcelToWutta(FromExcelHandler, ToWuttaHandler): + source_title = "My Spreadsheet" + See also :meth:`get_title()` and :meth:`get_target_title()`. """ if hasattr(self, "source_title"): @@ -221,6 +235,20 @@ class ImportHandler(GenericHandler): """ Returns the display title for the data target. + By default this returns :attr:`target_key`, but this can be + overriden by class attribute. + + Base class can define ``generic_target_title`` to provide a + new default:: + + class ToExcelHandler(ImportHandler): + generic_target_title = "Excel File" + + Subclass can define ``target_title`` to be explicit:: + + class FromWuttaToExcel(FromWuttaHandler, ToExcelHandler): + target_title = "My Spreadsheet" + See also :meth:`get_title()` and :meth:`get_source_title()`. """ if hasattr(self, "target_title"): @@ -538,9 +566,129 @@ class FromFileHandler(ImportHandler): super().process_data(*keys, **kwargs) +class FromSqlalchemyHandler(ImportHandler): + """ + Base class for import/export handlers using SQLAlchemy ORM (DB) as + data source. + + This is meant to be used with importers/exporters which inherit + from :class:`~wuttasync.importing.base.FromSqlalchemy`. It will + set the + :attr:`~wuttasync.importing.base.FromSqlalchemy.source_session` + attribute when making them; cf. :meth:`get_importer_kwargs()`. + + This is the base class for :class:`FromWuttaHandler`, but can be + used with any database. + + See also :class:`ToSqlalchemyHandler`. + """ + + source_session = None + """ + Reference to the :term:`db session` for data source. + + This will be ``None`` unless a transaction is running. + """ + + def begin_source_transaction(self): + """ + This calls :meth:`make_source_session()` and assigns the + result to :attr:`source_session`. + """ + self.source_session = self.make_source_session() + + def commit_source_transaction(self): + """ + This commits and closes :attr:`source_session`. + """ + self.source_session.commit() + self.source_session.close() + self.source_session = None + + def rollback_source_transaction(self): + """ + This rolls back, then closes :attr:`source_session`. + """ + self.source_session.rollback() + self.source_session.close() + self.source_session = None + + def make_source_session(self): + """ + Make and return a new :term:`db session` for the data source. + + Default logic is not implemented; subclass must override. + + :returns: :class:`~sqlalchemy.orm.Session` instance + """ + raise NotImplementedError + + def get_importer_kwargs(self, key, **kwargs): + """ + This modifies the new importer kwargs to add: + + * ``source_session`` - reference to :attr:`source_session` + + See also docs for parent method, + :meth:`~ImportHandler.get_importer_kwargs()`. + """ + kwargs = super().get_importer_kwargs(key, **kwargs) + kwargs["source_session"] = self.source_session + return kwargs + + +class FromWuttaHandler(FromSqlalchemyHandler): + """ + Handler for import/export which uses Wutta ORM (:term:`app + database`) as data source. + + This inherits from :class:`FromSqlalchemyHandler`. + + See also :class:`ToWuttaHandler`. + """ + + source_key = "wutta" + "" # nb. suppress docs + + def get_source_title(self): + """ + This overrides default logic to use + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.get_title()` + as the default value. + + Subclass can still define + :attr:`~wuttasync.importing.handlers.ImportHandler.source_title` + (or + :attr:`~wuttasync.importing.handlers.ImportHandler.generic_source_title`) + to customize. + + See also docs for parent method: + :meth:`~wuttasync.importing.handlers.ImportHandler.get_source_title()` + """ + if hasattr(self, "source_title"): + return self.source_title + if hasattr(self, "generic_source_title"): + return self.generic_source_title + return self.app.get_title() + + def make_source_session(self): + """ + This calls + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` + and returns it. + """ + return self.app.make_session() + + class ToSqlalchemyHandler(ImportHandler): """ - Handler for import/export which targets a SQLAlchemy ORM (DB). + Base class for import/export handlers which target a SQLAlchemy + ORM (DB). + + This is the base class for :class:`ToWuttaHandler`, but can be + used with any database. + + See also :class:`FromSqlalchemyHandler`. """ target_session = None @@ -591,3 +739,37 @@ class ToSqlalchemyHandler(ImportHandler): kwargs = super().get_importer_kwargs(key, **kwargs) kwargs.setdefault("target_session", self.target_session) return kwargs + + +class ToWuttaHandler(ToSqlalchemyHandler): + """ + Handler for import/export which targets Wutta ORM (:term:`app + database`). + + This inherits from :class:`ToSqlalchemyHandler`. + + See also :class:`FromWuttaHandler`. + """ + + target_key = "wutta" + "" # nb. suppress docs + + def get_target_title(self): # pylint: disable=empty-docstring + """ """ + # nb. we override parent to use app title as default + if hasattr(self, "target_title"): + return self.target_title + if hasattr(self, "generic_target_title"): + return self.generic_target_title + return self.app.get_title() + + def make_target_session(self): + """ + Call + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` + and return the result. + + :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session` + instance. + """ + return self.app.make_session() diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py new file mode 100644 index 0000000..53c25fa --- /dev/null +++ b/src/wuttasync/importing/versions.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Importing Versions + +This is a special type of import, only relevant when data versioning +is enabled. + +See the handler class for more info: :class:`FromWuttaToVersions` +""" + +from collections import OrderedDict + +from sqlalchemy_utils.functions import get_primary_keys + +from wuttjamaican.db.util import make_topo_sortkey + +from .handlers import FromWuttaHandler, ToWuttaHandler +from .wutta import FromWuttaMirror +from .model import ToWutta + + +class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler): + """ + Handler for Wutta -> Versions import. + + The purpose of this is to ensure version tables accurately reflect + the current "live" data set, for given table(s). It is only + relevant/usable if versioning is configured and enabled. For more + on that see :doc:`wutta-continuum:index`. + + For a given import model, the source is the "live" table, target + is the "version" table - both in the same :term:`app database`. + + When reading data from the target side, it only grabs the "latest" + (valid) version record for each comparison to source. + + When changes are needed, instead of updating the existing version + record, it always writes a new version record. + + This handler will dynamically create importers for all versioned + models in the :term:`app model`; see + :meth:`make_importer_factory()`. + """ + + target_key = "versions" + target_title = "Versions" + + continuum_uow = None + """ + Reference to the + :class:`sqlalchemy-continuum:`sqlalchemy_continuum.UnitOfWork` + created (by the SQLAlchemy-Continuum ``versioning_manager``) when + the transaction begins. + + See also :attr:`continuum_txn` and + :meth:`begin_target_transaction()`. + """ + + continuum_txn = None + """ + Reference to the SQLAlchemy-Continuum ``transaction`` record, to + which any new version records will associate (if needed). + + This transaction will track the effective user responsible for + the change(s), their client IP, and timestamp. + + This reference is passed along to the importers as well (as + :attr:`~FromWuttaToVersionBase.continuum_txn`) via + :meth:`get_importer_kwargs()`. + + See also :attr:`continuum_uow`. + """ + + def begin_target_transaction(self): + # pylint: disable=line-too-long + """ + In addition to normal logic, this does some setup for + SQLAlchemy-Continuum: + + It establishes a "unit of work" by calling + :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.VersioningManager.unit_of_work()`, + assigning the result to :attr:`continuum_uow`. + + It then calls + :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()` + and assigns that to :attr:`continuum_txn`. + + See also docs for parent method: + :meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()` + """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + super().begin_target_transaction() + + self.continuum_uow = continuum.versioning_manager.unit_of_work( + self.target_session + ) + self.continuum_txn = self.continuum_uow.create_transaction(self.target_session) + + def get_importer_kwargs(self, key, **kwargs): + """ + This modifies the new importer kwargs to add: + + * ``continuum_txn`` - reference to :attr:`continuum_txn` + + See also docs for parent method: + :meth:`~wuttasync.importing.handlers.ImportHandler.get_importer_kwargs()` + """ + kwargs = super().get_importer_kwargs(key, **kwargs) + kwargs["continuum_txn"] = self.continuum_txn + return kwargs + + # TODO: pylint (correctly) flags this as duplicate code, matching + # on the wuttasync.importing.csv module - should fix? + def define_importers(self): + """ + This overrides typical (manual) importer definition, instead + generating importers for all versioned models. + + It will inspect the :term:`app model` and call + :meth:`make_importer_factory()` for each model found, keeping + only the valid importers. + + See also the docs for parent method: + :meth:`~wuttasync.importing.handlers.ImportHandler.define_importers()` + """ + model = self.app.model + importers = {} + + # pylint: disable=duplicate-code + # mostly try to make an importer for every data model + for name in dir(model): + cls = getattr(model, name) + if ( + isinstance(cls, type) + and issubclass(cls, model.Base) + and cls is not model.Base + ): + # only keep "good" importers, i.e. for versioned models + if factory := self.make_importer_factory(cls, name): + importers[name] = factory + + # sort importers according to schema topography + topo_sortkey = make_topo_sortkey(model) + importers = OrderedDict( + [(name, importers[name]) for name in sorted(importers, key=topo_sortkey)] + ) + + return importers + + def make_importer_factory(self, model_class, name): + """ + Try to generate a new :term:`importer` class for the given + :term:`data model`. This is called by + :meth:`define_importers()`. + + If the provided ``model_class`` is not versioned, this will + fail and return ``None``. + + For a versioned model, the new importer class will inherit + from :class:`FromWuttaToVersionBase`. + + Its (target) + :attr:`~wuttasync.importing.base.Importer.model_class` will be + set to the **version** model. + + Its + :attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class` + will be set to the **normal** model. + + :param model_class: A (normal, not version) data model class. + + :param name: The "model name" for the importer. New class + name will be based on this, so e.g. ``Widget`` model name + becomes ``WidgetImporter`` class name. + + :returns: The new class, or ``None`` + """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + try: + version_class = continuum.version_class(model_class) + except continuum.exc.ClassNotVersioned: + return None + + return type( + f"{name}Importer", + (FromWuttaToVersionBase,), + { + "source_model_class": model_class, + "model_class": version_class, + "default_keys": list(get_primary_keys(model_class)), + }, + ) + + +class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): + """ + Base importer class for Wutta -> Versions. + + This imports from + :class:`~wuttasync.importing.wutta.FromWuttaMirror` and + :class:`~wuttasync.importing.model.ToWutta`. + + The import handler will dynamically generate importers using this + base class; see + :meth:`~FromWuttaToVersions.make_importer_factory()`. + """ + + continuum_txn = None + """ + Reference to the handler's attribute of the same name: + :attr:`~FromWuttaToVersions.continuum_txn` + + This is the SQLAlchemy-Continuum ``transaction`` record, to which + any new version records will associate (if needed). + + This transaction will track the effective user responsible for + the change(s), their client IP, and timestamp. + """ + + def get_simple_fields(self): # pylint: disable=empty-docstring + """ """ + fields = super().get_simple_fields() + unwanted = ["transaction_id", "operation_type", "end_transaction_id"] + fields = [field for field in fields if field not in unwanted] + return fields + + def get_target_query(self, source_data=None): + """ + This modifies the normal query to ensure we only get the + "latest valid" version for each record, for comparison to + source. + + .. note:: + + In some cases, it still may be possible for multiple + "latest" versions to match for a given record. This means + inconsistent data; a warning should be logged if so, and + you must track it down... + + See also docs for parent method: + :meth:`~wuttasync.importing.base.ToSqlalchemy.get_target_query()` + """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + # pylint: disable=singleton-comparison + return ( + self.target_session.query(self.model_class) + .filter(self.model_class.end_transaction_id == None) + .filter(self.model_class.operation_type != continuum.Operation.DELETE) + ) + + def normalize_target_object(self, obj): # pylint: disable=empty-docstring + """ """ + data = super().normalize_target_object(obj) + + # we want to add the original version object to normalized + # data, so we can access it later for updating if needed. but + # this method is called for *both* sides (source+target) since + # this is a "mirrored" importer. so we must check the type + # and only cache true versions, ignore "normal" objects. + if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type + obj, self.model_class + ): + data["_version"] = obj + + return data + + def make_version( # pylint: disable=missing-function-docstring + self, source_data, operation_type + ): + key = self.get_record_key(source_data) + with self.target_session.no_autoflush: + version = self.make_empty_object(key) + self.populate(version, source_data) + version.transaction = self.continuum_txn + version.operation_type = operation_type + self.target_session.add(version) + return version + + def populate(self, obj, data): # pylint: disable=missing-function-docstring + keys = self.get_keys() + for field in self.get_simple_fields(): + if field not in keys and field in data and field in self.fields: + setattr(obj, field, data[field]) + + def create_target_object(self, key, source_data): # pylint: disable=empty-docstring + """ """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + return self.make_version(source_data, continuum.Operation.INSERT) + + def update_target_object( # pylint: disable=empty-docstring + self, obj, source_data, target_data=None + ): + """ """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + # when we "update" it always involves making a *new* version + # record. but that requires actually updating the "previous" + # version to indicate the new version's transaction. + prev_version = target_data.pop("_version") + prev_version.end_transaction_id = self.continuum_txn.id + + return self.make_version(source_data, continuum.Operation.UPDATE) + + def delete_target_object(self, obj): # pylint: disable=empty-docstring + """ """ + import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel + + # nb. `obj` here is the existing/old version record; we update + # it to indicate the new version's transaction. + obj.end_transaction_id = self.continuum_txn.id + + # add new "DELETE" version record. values should be the same as + # for "previous" (existing/old) version. + source_data = self.normalize_target_object(obj) + return self.make_version(source_data, continuum.Operation.DELETE) diff --git a/src/wuttasync/importing/wutta.py b/src/wuttasync/importing/wutta.py index 9de4822..882f7df 100644 --- a/src/wuttasync/importing/wutta.py +++ b/src/wuttasync/importing/wutta.py @@ -21,37 +21,13 @@ # ################################################################################ """ -Wutta ⇄ Wutta import/export +Wutta → Wutta import/export """ -from .handlers import ToSqlalchemyHandler +from .base import FromSqlalchemyMirror -class ToWuttaHandler(ToSqlalchemyHandler): +class FromWuttaMirror(FromSqlalchemyMirror): # pylint: disable=abstract-method """ - Handler for import/export which targets Wutta ORM (:term:`app - database`). + Base class for Wutta -> Wutta data importers. """ - - target_key = "wutta" - "" # nb. suppress docs - - def get_target_title(self): # pylint: disable=empty-docstring - """ """ - # nb. we override parent to use app title as default - if hasattr(self, "target_title"): - return self.target_title - if hasattr(self, "generic_target_title"): - return self.generic_target_title - return self.app.get_title() - - def make_target_session(self): - """ - Call - :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` - and return the result. - - :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session` - instance. - """ - return self.app.make_session() diff --git a/tests/cli/test_import_versions.py b/tests/cli/test_import_versions.py new file mode 100644 index 0000000..ea1617d --- /dev/null +++ b/tests/cli/test_import_versions.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8; -*- + +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from wuttasync.cli import import_versions as mod, ImportCommandHandler + + +class TestImportCsv(TestCase): + + def test_basic(self): + params = { + "models": [], + "create": True, + "update": True, + "delete": False, + "dry_run": True, + } + ctx = MagicMock(params=params) + with patch.object(ImportCommandHandler, "run") as run: + mod.import_versions(ctx) + run.assert_called_once_with(params) diff --git a/tests/importing/test_base.py b/tests/importing/test_base.py index 08c37a2..c920ed6 100644 --- a/tests/importing/test_base.py +++ b/tests/importing/test_base.py @@ -2,6 +2,8 @@ from unittest.mock import patch +from sqlalchemy import orm + from wuttjamaican.testing import DataTestCase from wuttasync.importing import base as mod, ImportHandler, Orientation @@ -78,13 +80,31 @@ class TestImporter(DataTestCase): def test_get_keys(self): model = self.app.model + + # nb. get_keys() will cache the return value, so must + # re-create importer for each test + + # keys inspected from model by default imp = self.make_importer(model_class=model.Setting) self.assertEqual(imp.get_keys(), ["name"]) - with patch.multiple(imp, create=True, key="value"): - self.assertEqual(imp.get_keys(), ["value"]) - with patch.multiple(imp, create=True, keys=["foo", "bar"]): + imp = self.make_importer(model_class=model.User) + self.assertEqual(imp.get_keys(), ["uuid"]) + + # class may define 'keys' + imp = self.make_importer(model_class=model.User) + with patch.object(imp, "keys", new=["foo", "bar"], create=True): self.assertEqual(imp.get_keys(), ["foo", "bar"]) + # class may define 'key' + imp = self.make_importer(model_class=model.User) + with patch.object(imp, "key", new="whatever", create=True): + self.assertEqual(imp.get_keys(), ["whatever"]) + + # class may define 'default_keys' + imp = self.make_importer(model_class=model.User) + with patch.object(imp, "default_keys", new=["baz", "foo"]): + self.assertEqual(imp.get_keys(), ["baz", "foo"]) + def test_process_data(self): model = self.app.model imp = self.make_importer( @@ -651,6 +671,105 @@ class TestFromFile(DataTestCase): close.assert_called_once_with() +class TestQueryWrapper(DataTestCase): + + def test_basic(self): + model = self.app.model + + p1 = model.Person(full_name="John Doe") + self.session.add(p1) + p2 = model.Person(full_name="Jane Doe") + self.session.add(p2) + self.session.commit() + + # cannot get count via len(query), must use query.count() + query = self.session.query(model.Person) + self.assertEqual(query.count(), 2) + self.assertRaises(TypeError, len, query) + + # but can use len(wrapper) + wrapper = mod.QueryWrapper(query) + self.assertEqual(len(wrapper), 2) + + # iter(wrapper) should work too + people = [p for p in wrapper] + self.assertEqual(people, [p1, p2]) + people = [p for p in iter(wrapper)] + self.assertEqual(people, [p1, p2]) + people = [p for p in list(wrapper)] + self.assertEqual(people, [p1, p2]) + + +class TestFromSqlalchemy(DataTestCase): + + def setUp(self): + self.setup_db() + self.handler = ImportHandler(self.config) + + def make_importer(self, **kwargs): + kwargs.setdefault("handler", self.handler) + return mod.FromSqlalchemy(self.config, **kwargs) + + def test_get_source_query(self): + model = self.app.model + imp = self.make_importer( + source_model_class=model.Upgrade, source_session=self.session + ) + query = imp.get_source_query() + self.assertIsInstance(query, orm.Query) + self.assertEqual(len(query.selectable.froms), 1) + table = query.selectable.froms[0] + self.assertEqual(table.name, "upgrade") + + def test_get_source_objects(self): + model = self.app.model + + user1 = model.User(username="fred") + self.session.add(user1) + user2 = model.User(username="bettie") + self.session.add(user2) + self.session.commit() + + imp = self.make_importer( + source_model_class=model.User, source_session=self.session + ) + result = imp.get_source_objects() + self.assertIsInstance(result, mod.QueryWrapper) + self.assertEqual(len(result), 2) + self.assertEqual(list(result), [user1, user2]) + + +class TestFromSqlalchemyMirror(DataTestCase): + + def setUp(self): + self.setup_db() + self.handler = ImportHandler(self.config) + + def make_importer(self, **kwargs): + kwargs.setdefault("handler", self.handler) + return mod.FromSqlalchemyMirror(self.config, **kwargs) + + def test_source_model_class(self): + model = self.app.model + + # source_model_class will mirror model_class + imp = self.make_importer(model_class=model.Upgrade) + self.assertIs(imp.model_class, model.Upgrade) + self.assertIs(imp.source_model_class, model.Upgrade) + + def test_normalize_source_object(self): + model = self.app.model + imp = self.make_importer(model_class=model.Upgrade) + upgrade = model.Upgrade() + + # normalize_source_object() should invoke normalize_target_object() + with patch.object(imp, "normalize_target_object") as normalize_target_object: + normalize_target_object.return_value = 42 + result = imp.normalize_source_object(upgrade) + self.assertEqual(result, 42) + normalize_target_object.assert_called_once_with(upgrade) + + class TestToSqlalchemy(DataTestCase): def setUp(self): diff --git a/tests/importing/test_handlers.py b/tests/importing/test_handlers.py index 9bd0157..a6df032 100644 --- a/tests/importing/test_handlers.py +++ b/tests/importing/test_handlers.py @@ -213,6 +213,97 @@ class TestFromFileHandler(DataTestCase): process_data.assert_called_once_with(input_file_dir=self.tempdir) +class TestFromSqlalchemyHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.FromSqlalchemyHandler(self.config, **kwargs) + + def test_make_source_session(self): + handler = self.make_handler() + self.assertRaises(NotImplementedError, handler.make_source_session) + + def test_begin_source_transaction(self): + handler = self.make_handler() + self.assertIsNone(handler.source_session) + with patch.object(handler, "make_source_session", return_value=self.session): + handler.begin_source_transaction() + self.assertIs(handler.source_session, self.session) + + def test_commit_source_transaction(self): + model = self.app.model + handler = self.make_handler() + handler.source_session = self.session + self.assertEqual(self.session.query(model.User).count(), 0) + + # nb. do not commit this yet + user = model.User(username="fred") + self.session.add(user) + + self.assertTrue(self.session.in_transaction()) + self.assertIn(user, self.session) + handler.commit_source_transaction() + self.assertIsNone(handler.source_session) + self.assertFalse(self.session.in_transaction()) + self.assertNotIn(user, self.session) # hm, surprising? + self.assertEqual(self.session.query(model.User).count(), 1) + + def test_rollback_source_transaction(self): + model = self.app.model + handler = self.make_handler() + handler.source_session = self.session + self.assertEqual(self.session.query(model.User).count(), 0) + + # nb. do not commit this yet + user = model.User(username="fred") + self.session.add(user) + + self.assertTrue(self.session.in_transaction()) + self.assertIn(user, self.session) + handler.rollback_source_transaction() + self.assertIsNone(handler.source_session) + self.assertFalse(self.session.in_transaction()) + self.assertNotIn(user, self.session) + self.assertEqual(self.session.query(model.User).count(), 0) + + def test_get_importer_kwargs(self): + handler = self.make_handler() + handler.source_session = self.session + kw = handler.get_importer_kwargs("User") + self.assertIn("source_session", kw) + self.assertIs(kw["source_session"], self.session) + + +class TestFromWuttaHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.FromWuttaHandler(self.config, **kwargs) + + def test_get_source_title(self): + handler = self.make_handler() + + # uses app title by default + self.config.setdefault("wutta.app_title", "What About This") + self.assertEqual(handler.get_source_title(), "What About This") + + # or generic default if present + handler.generic_source_title = "WHATABOUTTHIS" + self.assertEqual(handler.get_source_title(), "WHATABOUTTHIS") + + # but prefer specific title if present + handler.source_title = "what_about_this" + self.assertEqual(handler.get_source_title(), "what_about_this") + + def test_make_source_session(self): + handler = self.make_handler() + + # makes "new" (mocked in our case) app session + with patch.object(self.app, "make_session") as make_session: + make_session.return_value = self.session + session = handler.make_source_session() + make_session.assert_called_once_with() + self.assertIs(session, self.session) + + class TestToSqlalchemyHandler(DataTestCase): def make_handler(self, **kwargs): @@ -256,3 +347,34 @@ class TestToSqlalchemyHandler(DataTestCase): kw = handler.get_importer_kwargs("Setting") self.assertIn("target_session", kw) self.assertIs(kw["target_session"], self.session) + + +class TestToWuttaHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.ToWuttaHandler(self.config, **kwargs) + + def test_get_target_title(self): + handler = self.make_handler() + + # uses app title by default + self.config.setdefault("wutta.app_title", "What About This") + self.assertEqual(handler.get_target_title(), "What About This") + + # or generic default if present + handler.generic_target_title = "WHATABOUTTHIS" + self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS") + + # but prefer specific title if present + handler.target_title = "what_about_this" + self.assertEqual(handler.get_target_title(), "what_about_this") + + def test_make_target_session(self): + handler = self.make_handler() + + # makes "new" (mocked in our case) app session + with patch.object(self.app, "make_session") as make_session: + make_session.return_value = self.session + session = handler.make_target_session() + make_session.assert_called_once_with() + self.assertIs(session, self.session) diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py new file mode 100644 index 0000000..2067f93 --- /dev/null +++ b/tests/importing/test_versions.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8; -*- + +from sqlalchemy import orm +import sqlalchemy_continuum as continuum + +from wuttjamaican.util import make_true_uuid +from wutta_continuum.testing import VersionTestCase + +from wuttasync.importing import versions as mod, Importer + + +class TestFromWuttaToVersions(VersionTestCase): + + def make_handler(self, **kwargs): + return mod.FromWuttaToVersions(self.config, **kwargs) + + def test_begin_target_transaction(self): + model = self.app.model + txncls = continuum.transaction_class(model.User) + + handler = self.make_handler() + self.assertIsNone(handler.continuum_uow) + self.assertIsNone(handler.continuum_txn) + + handler.begin_target_transaction() + self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork) + self.assertIsInstance(handler.continuum_txn, txncls) + + def test_get_importer_kwargs(self): + handler = self.make_handler() + handler.begin_target_transaction() + + kw = handler.get_importer_kwargs("User") + self.assertIn("continuum_txn", kw) + self.assertIs(kw["continuum_txn"], handler.continuum_txn) + + def test_make_importer_factory(self): + model = self.app.model + handler = self.make_handler() + + # versioned class + factory = handler.make_importer_factory(model.User, "User") + self.assertTrue(issubclass(factory, mod.FromWuttaToVersionBase)) + self.assertIs(factory.source_model_class, model.User) + self.assertIs(factory.model_class, continuum.version_class(model.User)) + + # non-versioned + factory = handler.make_importer_factory(model.Upgrade, "Upgrade") + self.assertIsNone(factory) + + def test_define_importers(self): + handler = self.make_handler() + + importers = handler.define_importers() + self.assertIn("User", importers) + self.assertIn("Person", importers) + self.assertNotIn("Upgrade", importers) + + +class UserImporter(mod.FromWuttaToVersionBase): + + @property + def model_class(self): + model = self.app.model + return model.User + + +class TestFromWuttaToVersionBase(VersionTestCase): + + def make_importer(self, model_class=None, **kwargs): + imp = mod.FromWuttaToVersionBase(self.config, **kwargs) + if model_class: + imp.model_class = model_class + return imp + + def test_get_simple_fields(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + # first confirm what a "normal" importer would do + imp = Importer(self.config, model_class=vercls) + fields = imp.get_simple_fields() + self.assertIn("username", fields) + self.assertIn("person_uuid", fields) + self.assertIn("transaction_id", fields) + self.assertIn("operation_type", fields) + self.assertIn("end_transaction_id", fields) + + # now test what the "version" importer does + imp = self.make_importer(model_class=vercls) + fields = imp.get_simple_fields() + self.assertIn("username", fields) + self.assertIn("person_uuid", fields) + self.assertNotIn("transaction_id", fields) + self.assertNotIn("operation_type", fields) + self.assertNotIn("end_transaction_id", fields) + + def test_get_target_query(self): + model = self.app.model + vercls = continuum.version_class(model.User) + imp = self.make_importer(model_class=vercls, target_session=self.session) + + # TODO: not sure what else to test here.. + query = imp.get_target_query() + self.assertIsInstance(query, orm.Query) + + def test_normalize_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + imp = self.make_importer(model_class=vercls) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + version = user.versions[0] + + # version object should be embedded in data dict + data = imp.normalize_target_object(version) + self.assertIsInstance(data, dict) + self.assertIn("_version", data) + self.assertIs(data["_version"], version) + + # but normal object is not embedded + data = imp.normalize_target_object(user) + self.assertIsInstance(data, dict) + self.assertNotIn("_version", data) + + def test_make_version(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + data = {"uuid": user.uuid, "username": "freddie"} + version = imp.make_version(data, continuum.Operation.UPDATE) + self.assertIsInstance(version, vercls) + self.assertEqual(version.uuid, user.uuid) + self.assertEqual(version.username, "freddie") + self.assertIn(version, self.session) + self.assertIs(version.transaction, imp.continuum_txn) + self.assertEqual(version.operation_type, continuum.Operation.UPDATE) + + def test_create_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + source_data = {"uuid": make_true_uuid(), "username": "bettie"} + self.assertEqual(self.session.query(vercls).count(), 0) + version = imp.create_target_object((source_data["uuid"], 1), source_data) + self.assertEqual(self.session.query(vercls).count(), 1) + self.assertEqual(version.transaction_id, imp.continuum_txn.id) + self.assertEqual(version.operation_type, continuum.Operation.INSERT) + self.assertIsNone(version.end_transaction_id) + + def test_update_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + version1 = user.versions[0] + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + source_data = {"uuid": user.uuid, "username": "freddie"} + target_data = imp.normalize_target_object(version1) + self.assertEqual(self.session.query(vercls).count(), 1) + self.assertIsNone(version1.end_transaction_id) + version2 = imp.update_target_object( + version1, source_data, target_data=target_data + ) + self.assertEqual(self.session.query(vercls).count(), 2) + self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.operation_type, continuum.Operation.UPDATE) + self.assertIsNone(version2.end_transaction_id) + + def test_delete_target_object(self): + model = self.app.model + vercls = continuum.version_class(model.User) + + user = model.User(username="fred") + self.session.add(user) + self.session.commit() + version1 = user.versions[0] + + handler = mod.FromWuttaToVersions(self.config) + handler.begin_target_transaction() + handler.target_session.close() + handler.target_session = self.session + + imp = self.make_importer( + model_class=vercls, + fields=["uuid", "username"], + keys=("uuid",), + target_session=self.session, + continuum_txn=handler.continuum_txn, + ) + + self.assertEqual(self.session.query(vercls).count(), 1) + self.assertIsNone(version1.end_transaction_id) + version2 = imp.delete_target_object(version1) + self.assertEqual(self.session.query(vercls).count(), 2) + self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.transaction_id, imp.continuum_txn.id) + self.assertEqual(version2.operation_type, continuum.Operation.DELETE) + self.assertIsNone(version2.end_transaction_id) diff --git a/tests/importing/test_wutta.py b/tests/importing/test_wutta.py index 4d6fdd2..1533605 100644 --- a/tests/importing/test_wutta.py +++ b/tests/importing/test_wutta.py @@ -1,38 +1,3 @@ # -*- coding: utf-8; -*- -from unittest.mock import patch - -from wuttjamaican.testing import DataTestCase - from wuttasync.importing import wutta as mod - - -class TestToWuttaHandler(DataTestCase): - - def make_handler(self, **kwargs): - return mod.ToWuttaHandler(self.config, **kwargs) - - def test_get_target_title(self): - handler = self.make_handler() - - # uses app title by default - self.config.setdefault("wutta.app_title", "What About This") - self.assertEqual(handler.get_target_title(), "What About This") - - # or generic default if present - handler.generic_target_title = "WHATABOUTTHIS" - self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS") - - # but prefer specific title if present - handler.target_title = "what_about_this" - self.assertEqual(handler.get_target_title(), "what_about_this") - - def test_make_target_session(self): - handler = self.make_handler() - - # makes "new" (mocked in our case) app session - with patch.object(self.app, "make_session") as make_session: - make_session.return_value = self.session - session = handler.make_target_session() - make_session.assert_called_once_with() - self.assertIs(session, self.session)