diff --git a/docs/api/wuttasync.cli.import_versions.rst b/docs/api/wuttasync.cli.import_versions.rst
new file mode 100644
index 0000000..aeb8227
--- /dev/null
+++ b/docs/api/wuttasync.cli.import_versions.rst
@@ -0,0 +1,6 @@
+
+``wuttasync.cli.import_versions``
+=================================
+
+.. automodule:: wuttasync.cli.import_versions
+ :members:
diff --git a/docs/api/wuttasync.importing.versions.rst b/docs/api/wuttasync.importing.versions.rst
new file mode 100644
index 0000000..aa970a1
--- /dev/null
+++ b/docs/api/wuttasync.importing.versions.rst
@@ -0,0 +1,6 @@
+
+``wuttasync.importing.versions``
+================================
+
+.. automodule:: wuttasync.importing.versions
+ :members:
diff --git a/docs/conf.py b/docs/conf.py
index 2b47550..7826856 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -31,6 +31,13 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
intersphinx_mapping = {
"python": ("https://docs.python.org/3/", None),
"rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None),
+ "sqlalchemy": ("http://docs.sqlalchemy.org/en/latest/", None),
+ "sqlalchemy-continuum": (
+ "https://sqlalchemy-continuum.readthedocs.io/en/latest/",
+ None,
+ ),
+ "sqlalchemy-utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None),
+ "wutta-continuum": ("https://docs.wuttaproject.org/wutta-continuum/", None),
"wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None),
}
diff --git a/docs/index.rst b/docs/index.rst
index 9eb2d93..6fe554a 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -73,10 +73,12 @@ cf. :doc:`rattail-manual:data/sync/index`.
api/wuttasync.cli
api/wuttasync.cli.base
api/wuttasync.cli.import_csv
+ api/wuttasync.cli.import_versions
api/wuttasync.importing
api/wuttasync.importing.base
api/wuttasync.importing.csv
api/wuttasync.importing.handlers
api/wuttasync.importing.model
+ api/wuttasync.importing.versions
api/wuttasync.importing.wutta
api/wuttasync.util
diff --git a/docs/narr/cli/builtin.rst b/docs/narr/cli/builtin.rst
index 0630c94..ac6fb14 100644
--- a/docs/narr/cli/builtin.rst
+++ b/docs/narr/cli/builtin.rst
@@ -25,3 +25,24 @@ types may not behave as expected etc.
Defined in: :mod:`wuttasync.cli.import_csv`
.. program-output:: wutta import-csv --help
+
+
+.. _wutta-import-versions:
+
+``wutta import-versions``
+-------------------------
+
+Import latest data to version tables, for the Wutta :term:`app
+database`.
+
+The purpose of this is to ensure version tables accurately reflect
+the current "live" data set, for given table(s). It is only
+relevant/usable if versioning is configured and enabled. For more
+on that see :doc:`wutta-continuum:index`.
+
+This command can check/update version tables for any versioned class
+in the :term:`app model`.
+
+Defined in: :mod:`wuttasync.cli.import_versions`
+
+.. program-output:: wutta import-versions --help
diff --git a/pyproject.toml b/pyproject.toml
index a48b949..cff065a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,7 +34,7 @@ dependencies = [
[project.optional-dependencies]
docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
-tests = ["pylint", "pytest", "pytest-cov", "tox"]
+tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"]
[project.entry-points."wutta.typer_imports"]
diff --git a/src/wuttasync/cli/__init__.py b/src/wuttasync/cli/__init__.py
index c77a4e2..0d88ed4 100644
--- a/src/wuttasync/cli/__init__.py
+++ b/src/wuttasync/cli/__init__.py
@@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
-# Copyright © 2024 Lance Edgar
+# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@@ -34,3 +34,4 @@ from .base import import_command, file_import_command, ImportCommandHandler
# nb. must bring in all modules for discovery to work
from . import import_csv
+from . import import_versions
diff --git a/src/wuttasync/cli/import_versions.py b/src/wuttasync/cli/import_versions.py
new file mode 100644
index 0000000..f1d0481
--- /dev/null
+++ b/src/wuttasync/cli/import_versions.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+# WuttaSync -- Wutta Framework for data import/export and real-time sync
+# Copyright © 2024-2025 Lance Edgar
+#
+# This file is part of Wutta Framework.
+#
+# Wutta Framework is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# Wutta Framework is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Wutta Framework. If not, see .
+#
+################################################################################
+"""
+See also: :ref:`wutta-import-versions`
+"""
+
+import sys
+
+import rich
+import typer
+
+from wuttjamaican.cli import wutta_typer
+
+from .base import import_command, ImportCommandHandler
+
+
+@wutta_typer.command()
+@import_command
+def import_versions(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
+ """
+ Import latest data to version tables, for Wutta DB
+ """
+ config = ctx.parent.wutta_config
+ app = config.get_app()
+
+ # warn/exit if libs are not installed
+ try:
+ import wutta_continuum # pylint: disable=import-outside-toplevel,unused-import
+ except ImportError: # pragma: no cover
+ rich.print(
+ "\n\t[bold yellow]Wutta-Continum is not installed![/bold yellow]\n"
+ "\n\tIf you want it, run: pip install Wutta-Continuum\n"
+ )
+ sys.exit(1)
+
+ # warn/exit if feature disabled
+ if not app.continuum_is_enabled(): # pragma: no cover
+ rich.print(
+ "\n\t[bold yellow]Wutta-Continum is not enabled![/bold yellow]\n"
+ "\n\tIf you want it, see: https://docs.wuttaproject.org/wutta-continuum/\n"
+ )
+ sys.exit(1)
+
+ handler = ImportCommandHandler(
+ config, import_handler="wuttasync.importing.versions:FromWuttaToVersions"
+ )
+ handler.run(ctx.params)
diff --git a/src/wuttasync/importing/__init__.py b/src/wuttasync/importing/__init__.py
index 03a421f..545cbb9 100644
--- a/src/wuttasync/importing/__init__.py
+++ b/src/wuttasync/importing/__init__.py
@@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
-# Copyright © 2024 Lance Edgar
+# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@@ -32,7 +32,7 @@ And some :term:`import handler` base classes:
* :class:`~wuttasync.importing.handlers.ImportHandler`
* :class:`~wuttasync.importing.handlers.FromFileHandler`
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
-* :class:`~wuttasync.importing.wutta.ToWuttaHandler`
+* :class:`~wuttasync.importing.handlers.ToWuttaHandler`
And some :term:`importer` base classes:
@@ -42,7 +42,12 @@ And some :term:`importer` base classes:
* :class:`~wuttasync.importing.model.ToWutta`
"""
-from .handlers import Orientation, ImportHandler, FromFileHandler, ToSqlalchemyHandler
+from .handlers import (
+ Orientation,
+ ImportHandler,
+ FromFileHandler,
+ ToSqlalchemyHandler,
+ ToWuttaHandler,
+)
from .base import Importer, FromFile, ToSqlalchemy
from .model import ToWutta
-from .wutta import ToWuttaHandler
diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py
index 629ead6..ca0718e 100644
--- a/src/wuttasync/importing/base.py
+++ b/src/wuttasync/importing/base.py
@@ -184,6 +184,19 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
:meth:`get_target_cache()`.
"""
+ default_keys = None
+ """
+ In certain edge cases, the importer class must declare its key
+ list without using :attr:`keys`.
+
+ (As of now this only happens with
+ :class:`~wuttasync.importing.versions.FromWuttaToVersions` which
+ must dynamically create importer classes.)
+
+ If applicable, this value is used as fallback for
+ :meth:`get_keys()`.
+ """
+
max_create = None
max_update = None
max_delete = None
@@ -323,19 +336,54 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
def get_keys(self):
"""
- Must return the key field(s) for use with import/export.
+ Retrieve the list of key field(s) for use with import/export.
+ The result is cached, so the key list is only calculated once.
+
+ Many importers have just one key field, but we always assume a
+ key *list* - so this often is a list with just one field.
All fields in this list should also be found in the output for
:meth:`get_fields()`.
+ Many importers will declare this via :attr:`keys` (or
+ :attr:`key`) static attribute::
+
+ class SprocketImporter(Importer):
+
+ # nb. all these examples work the same
+
+ # 'keys' is the preferred attribute
+ keys = ("sprocket_id",) # <-- the "canonical" way
+ keys = ["sprocket_id"]
+ keys = "sprocket_id"
+
+ # 'key' is not preferred, but works
+ key = ("sprocket_id",)
+ key = "sprocket_id"
+
+ If neither ``keys`` nor ``key`` is set, as a special case
+ :attr:`default_keys` is used if set.
+
+ If no keys were declared, the list is inspected from the model
+ class via
+ :func:`sqlalchemy-utils:sqlalchemy_utils.functions.get_primary_keys()`.
+
+ In any case, the determination is made only once. This method
+ also *sets* :attr:`keys` on the instance, so it will return
+ that as-is for subsequent calls.
+
:returns: List of "key" field names.
"""
keys = None
+
# nb. prefer 'keys' but use 'key' as fallback
if "keys" in self.__dict__:
keys = self.__dict__["keys"]
elif "key" in self.__dict__:
keys = self.__dict__["key"]
+ else:
+ keys = self.default_keys
+
if keys:
if isinstance(keys, str):
keys = self.config.parse_list(keys)
@@ -1271,10 +1319,139 @@ class FromFile(Importer):
self.input_file.close()
+class QueryWrapper:
+ """
+ Simple wrapper for a SQLAlchemy query, to make it sort of behave
+ so that an importer can treat it as a data record list.
+
+ :param query: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
+ """
+
+ def __init__(self, query):
+ self.query = query
+
+ def __len__(self):
+ try:
+ return len(self.query)
+ except TypeError:
+ return self.query.count()
+
+ def __iter__(self):
+ return iter(self.query)
+
+
+class FromSqlalchemy(Importer): # pylint: disable=abstract-method
+ """
+ Base class for importer/exporter using SQL/ORM query as data
+ source.
+
+ Subclass should define :attr:`source_model_class` in which case
+ the source query is automatic. And/or override
+ :meth:`get_source_query()` to customize.
+
+ See also :class:`FromSqlalchemyMirror` and :class:`ToSqlalchemy`.
+ """
+
+ source_model_class = None
+ """
+ Reference to the :term:`data model` class representing the source.
+
+ This normally is a SQLAlchemy mapped class, e.g.
+ :class:`~wuttjamaican:wuttjamaican.db.model.base.Person` for
+ exporting from the Wutta People table.
+ """
+
+ source_session = None
+ """
+ Reference to the open :term:`db session` for the data source.
+
+ The importer must be given this reference when instantiated by the
+ :term:`import handler`. This is handled automatically if using
+ :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`.
+ """
+
+ def get_source_objects(self):
+ """
+ This method is responsible for fetching "raw" (non-normalized)
+ records from data source.
+
+ (See also the parent method docs for
+ :meth:`~wuttasync.importing.base.Importer.get_source_objects()`.)
+
+ It calls :meth:`get_source_query()` and then wraps that in a
+ :class:`QueryWrapper`, which is then returned.
+
+ Note that this method does not technically "retrieve" records
+ from the query; that happens automatically later.
+
+ :returns: :class:`QueryWrapper` for the source query
+ """
+ query = self.get_source_query()
+ return QueryWrapper(query)
+
+ def get_source_query(self):
+ """
+ This returns the SQL/ORM query used to fetch source
+ data. It is called from :meth:`get_source_objects()`.
+
+ Default logic just makes a simple ``SELECT * FROM TABLE`` kind
+ of query. Subclass can override as needed.
+
+ :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
+ """
+ return self.source_session.query(self.source_model_class)
+
+
+class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method
+ """
+ Special base class for when the source and target are effectively
+ mirrored, and can each be represented by the same :term:`data
+ model`.
+
+ The assumption is that SQLAlchemy ORM is used on both sides, even
+ though this base class only defines the source side (it inherits
+ from :class:`FromSqlalchemy`).
+
+ There are two main use cases for this:
+
+ * sync between app nodes
+ * sync version tables
+
+ When 2 app nodes are synced, the source and target are "the same"
+ in a schema sense, e.g. ``sprockets on node 01 => sprockets on
+ node 02``.
+
+ When version tables are synced, the same schema can be used for
+ the "live" table and the "version" table, e.g. ``sprockets =>
+ sprocket versions``.
+ """
+
+ @property
+ def source_model_class(self):
+ """
+ This returns the :attr:`~Importer.model_class` since source
+ and target must share common schema.
+ """
+ return self.model_class
+
+ def normalize_source_object(self, obj):
+ """
+ Since source/target share schema, there should be no tricky
+ normalization involved.
+
+ This calls :meth:`~Importer.normalize_target_object()` since
+ that logic should already be defined. This ensures the same
+ normalization is used on both sides.
+ """
+ return self.normalize_target_object(obj)
+
+
class ToSqlalchemy(Importer):
"""
Base class for importer/exporter which uses SQLAlchemy ORM on the
target side.
+
+ See also :class:`FromSqlalchemy`.
"""
caches_target = True
@@ -1312,6 +1489,8 @@ class ToSqlalchemy(Importer):
Returns an ORM query suitable to fetch existing objects from
the target side. This is called from
:meth:`get_target_objects()`.
+
+ :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
"""
return self.target_session.query(self.model_class)
diff --git a/src/wuttasync/importing/csv.py b/src/wuttasync/importing/csv.py
index ab0bf21..9190099 100644
--- a/src/wuttasync/importing/csv.py
+++ b/src/wuttasync/importing/csv.py
@@ -38,8 +38,7 @@ from wuttjamaican.db.util import make_topo_sortkey, UUID
from wuttjamaican.util import parse_bool
from .base import FromFile
-from .handlers import FromFileHandler
-from .wutta import ToWuttaHandler
+from .handlers import FromFileHandler, ToWuttaHandler
from .model import ToWutta
@@ -239,6 +238,8 @@ class FromCsvToSqlalchemyHandlerMixin:
"""
raise NotImplementedError
+ # TODO: pylint (correctly) flags this as duplicate code, matching
+ # on the wuttasync.importing.versions module - should fix?
def define_importers(self):
"""
This mixin overrides typical (manual) importer definition, and
@@ -252,6 +253,7 @@ class FromCsvToSqlalchemyHandlerMixin:
importers = {}
model = self.get_target_model()
+ # pylint: disable=duplicate-code
# mostly try to make an importer for every data model
for name in dir(model):
cls = getattr(model, name)
diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py
index e9c6ac3..c1f7595 100644
--- a/src/wuttasync/importing/handlers.py
+++ b/src/wuttasync/importing/handlers.py
@@ -209,6 +209,20 @@ class ImportHandler(GenericHandler):
"""
Returns the display title for the data source.
+ By default this returns :attr:`source_key`, but this can be
+ overriden by class attribute.
+
+ Base class can define ``generic_source_title`` to provide a
+ new default::
+
+ class FromExcelHandler(ImportHandler):
+ generic_source_title = "Excel File"
+
+ Subclass can define ``source_title`` to be explicit::
+
+ class FromExcelToWutta(FromExcelHandler, ToWuttaHandler):
+ source_title = "My Spreadsheet"
+
See also :meth:`get_title()` and :meth:`get_target_title()`.
"""
if hasattr(self, "source_title"):
@@ -221,6 +235,20 @@ class ImportHandler(GenericHandler):
"""
Returns the display title for the data target.
+ By default this returns :attr:`target_key`, but this can be
+ overriden by class attribute.
+
+ Base class can define ``generic_target_title`` to provide a
+ new default::
+
+ class ToExcelHandler(ImportHandler):
+ generic_target_title = "Excel File"
+
+ Subclass can define ``target_title`` to be explicit::
+
+ class FromWuttaToExcel(FromWuttaHandler, ToExcelHandler):
+ target_title = "My Spreadsheet"
+
See also :meth:`get_title()` and :meth:`get_source_title()`.
"""
if hasattr(self, "target_title"):
@@ -538,9 +566,129 @@ class FromFileHandler(ImportHandler):
super().process_data(*keys, **kwargs)
+class FromSqlalchemyHandler(ImportHandler):
+ """
+ Base class for import/export handlers using SQLAlchemy ORM (DB) as
+ data source.
+
+ This is meant to be used with importers/exporters which inherit
+ from :class:`~wuttasync.importing.base.FromSqlalchemy`. It will
+ set the
+ :attr:`~wuttasync.importing.base.FromSqlalchemy.source_session`
+ attribute when making them; cf. :meth:`get_importer_kwargs()`.
+
+ This is the base class for :class:`FromWuttaHandler`, but can be
+ used with any database.
+
+ See also :class:`ToSqlalchemyHandler`.
+ """
+
+ source_session = None
+ """
+ Reference to the :term:`db session` for data source.
+
+ This will be ``None`` unless a transaction is running.
+ """
+
+ def begin_source_transaction(self):
+ """
+ This calls :meth:`make_source_session()` and assigns the
+ result to :attr:`source_session`.
+ """
+ self.source_session = self.make_source_session()
+
+ def commit_source_transaction(self):
+ """
+ This commits and closes :attr:`source_session`.
+ """
+ self.source_session.commit()
+ self.source_session.close()
+ self.source_session = None
+
+ def rollback_source_transaction(self):
+ """
+ This rolls back, then closes :attr:`source_session`.
+ """
+ self.source_session.rollback()
+ self.source_session.close()
+ self.source_session = None
+
+ def make_source_session(self):
+ """
+ Make and return a new :term:`db session` for the data source.
+
+ Default logic is not implemented; subclass must override.
+
+ :returns: :class:`~sqlalchemy.orm.Session` instance
+ """
+ raise NotImplementedError
+
+ def get_importer_kwargs(self, key, **kwargs):
+ """
+ This modifies the new importer kwargs to add:
+
+ * ``source_session`` - reference to :attr:`source_session`
+
+ See also docs for parent method,
+ :meth:`~ImportHandler.get_importer_kwargs()`.
+ """
+ kwargs = super().get_importer_kwargs(key, **kwargs)
+ kwargs["source_session"] = self.source_session
+ return kwargs
+
+
+class FromWuttaHandler(FromSqlalchemyHandler):
+ """
+ Handler for import/export which uses Wutta ORM (:term:`app
+ database`) as data source.
+
+ This inherits from :class:`FromSqlalchemyHandler`.
+
+ See also :class:`ToWuttaHandler`.
+ """
+
+ source_key = "wutta"
+ "" # nb. suppress docs
+
+ def get_source_title(self):
+ """
+ This overrides default logic to use
+ :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.get_title()`
+ as the default value.
+
+ Subclass can still define
+ :attr:`~wuttasync.importing.handlers.ImportHandler.source_title`
+ (or
+ :attr:`~wuttasync.importing.handlers.ImportHandler.generic_source_title`)
+ to customize.
+
+ See also docs for parent method:
+ :meth:`~wuttasync.importing.handlers.ImportHandler.get_source_title()`
+ """
+ if hasattr(self, "source_title"):
+ return self.source_title
+ if hasattr(self, "generic_source_title"):
+ return self.generic_source_title
+ return self.app.get_title()
+
+ def make_source_session(self):
+ """
+ This calls
+ :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
+ and returns it.
+ """
+ return self.app.make_session()
+
+
class ToSqlalchemyHandler(ImportHandler):
"""
- Handler for import/export which targets a SQLAlchemy ORM (DB).
+ Base class for import/export handlers which target a SQLAlchemy
+ ORM (DB).
+
+ This is the base class for :class:`ToWuttaHandler`, but can be
+ used with any database.
+
+ See also :class:`FromSqlalchemyHandler`.
"""
target_session = None
@@ -591,3 +739,37 @@ class ToSqlalchemyHandler(ImportHandler):
kwargs = super().get_importer_kwargs(key, **kwargs)
kwargs.setdefault("target_session", self.target_session)
return kwargs
+
+
+class ToWuttaHandler(ToSqlalchemyHandler):
+ """
+ Handler for import/export which targets Wutta ORM (:term:`app
+ database`).
+
+ This inherits from :class:`ToSqlalchemyHandler`.
+
+ See also :class:`FromWuttaHandler`.
+ """
+
+ target_key = "wutta"
+ "" # nb. suppress docs
+
+ def get_target_title(self): # pylint: disable=empty-docstring
+ """ """
+ # nb. we override parent to use app title as default
+ if hasattr(self, "target_title"):
+ return self.target_title
+ if hasattr(self, "generic_target_title"):
+ return self.generic_target_title
+ return self.app.get_title()
+
+ def make_target_session(self):
+ """
+ Call
+ :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
+ and return the result.
+
+ :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
+ instance.
+ """
+ return self.app.make_session()
diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py
new file mode 100644
index 0000000..53c25fa
--- /dev/null
+++ b/src/wuttasync/importing/versions.py
@@ -0,0 +1,340 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+# WuttaSync -- Wutta Framework for data import/export and real-time sync
+# Copyright © 2024-2025 Lance Edgar
+#
+# This file is part of Wutta Framework.
+#
+# Wutta Framework is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# Wutta Framework is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Wutta Framework. If not, see .
+#
+################################################################################
+"""
+Importing Versions
+
+This is a special type of import, only relevant when data versioning
+is enabled.
+
+See the handler class for more info: :class:`FromWuttaToVersions`
+"""
+
+from collections import OrderedDict
+
+from sqlalchemy_utils.functions import get_primary_keys
+
+from wuttjamaican.db.util import make_topo_sortkey
+
+from .handlers import FromWuttaHandler, ToWuttaHandler
+from .wutta import FromWuttaMirror
+from .model import ToWutta
+
+
+class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
+ """
+ Handler for Wutta -> Versions import.
+
+ The purpose of this is to ensure version tables accurately reflect
+ the current "live" data set, for given table(s). It is only
+ relevant/usable if versioning is configured and enabled. For more
+ on that see :doc:`wutta-continuum:index`.
+
+ For a given import model, the source is the "live" table, target
+ is the "version" table - both in the same :term:`app database`.
+
+ When reading data from the target side, it only grabs the "latest"
+ (valid) version record for each comparison to source.
+
+ When changes are needed, instead of updating the existing version
+ record, it always writes a new version record.
+
+ This handler will dynamically create importers for all versioned
+ models in the :term:`app model`; see
+ :meth:`make_importer_factory()`.
+ """
+
+ target_key = "versions"
+ target_title = "Versions"
+
+ continuum_uow = None
+ """
+ Reference to the
+ :class:`sqlalchemy-continuum:`sqlalchemy_continuum.UnitOfWork`
+ created (by the SQLAlchemy-Continuum ``versioning_manager``) when
+ the transaction begins.
+
+ See also :attr:`continuum_txn` and
+ :meth:`begin_target_transaction()`.
+ """
+
+ continuum_txn = None
+ """
+ Reference to the SQLAlchemy-Continuum ``transaction`` record, to
+ which any new version records will associate (if needed).
+
+ This transaction will track the effective user responsible for
+ the change(s), their client IP, and timestamp.
+
+ This reference is passed along to the importers as well (as
+ :attr:`~FromWuttaToVersionBase.continuum_txn`) via
+ :meth:`get_importer_kwargs()`.
+
+ See also :attr:`continuum_uow`.
+ """
+
+ def begin_target_transaction(self):
+ # pylint: disable=line-too-long
+ """
+ In addition to normal logic, this does some setup for
+ SQLAlchemy-Continuum:
+
+ It establishes a "unit of work" by calling
+ :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.VersioningManager.unit_of_work()`,
+ assigning the result to :attr:`continuum_uow`.
+
+ It then calls
+ :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()`
+ and assigns that to :attr:`continuum_txn`.
+
+ See also docs for parent method:
+ :meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()`
+ """
+ import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
+
+ super().begin_target_transaction()
+
+ self.continuum_uow = continuum.versioning_manager.unit_of_work(
+ self.target_session
+ )
+ self.continuum_txn = self.continuum_uow.create_transaction(self.target_session)
+
+ def get_importer_kwargs(self, key, **kwargs):
+ """
+ This modifies the new importer kwargs to add:
+
+ * ``continuum_txn`` - reference to :attr:`continuum_txn`
+
+ See also docs for parent method:
+ :meth:`~wuttasync.importing.handlers.ImportHandler.get_importer_kwargs()`
+ """
+ kwargs = super().get_importer_kwargs(key, **kwargs)
+ kwargs["continuum_txn"] = self.continuum_txn
+ return kwargs
+
+ # TODO: pylint (correctly) flags this as duplicate code, matching
+ # on the wuttasync.importing.csv module - should fix?
+ def define_importers(self):
+ """
+ This overrides typical (manual) importer definition, instead
+ generating importers for all versioned models.
+
+ It will inspect the :term:`app model` and call
+ :meth:`make_importer_factory()` for each model found, keeping
+ only the valid importers.
+
+ See also the docs for parent method:
+ :meth:`~wuttasync.importing.handlers.ImportHandler.define_importers()`
+ """
+ model = self.app.model
+ importers = {}
+
+ # pylint: disable=duplicate-code
+ # mostly try to make an importer for every data model
+ for name in dir(model):
+ cls = getattr(model, name)
+ if (
+ isinstance(cls, type)
+ and issubclass(cls, model.Base)
+ and cls is not model.Base
+ ):
+ # only keep "good" importers, i.e. for versioned models
+ if factory := self.make_importer_factory(cls, name):
+ importers[name] = factory
+
+ # sort importers according to schema topography
+ topo_sortkey = make_topo_sortkey(model)
+ importers = OrderedDict(
+ [(name, importers[name]) for name in sorted(importers, key=topo_sortkey)]
+ )
+
+ return importers
+
+ def make_importer_factory(self, model_class, name):
+ """
+ Try to generate a new :term:`importer` class for the given
+ :term:`data model`. This is called by
+ :meth:`define_importers()`.
+
+ If the provided ``model_class`` is not versioned, this will
+ fail and return ``None``.
+
+ For a versioned model, the new importer class will inherit
+ from :class:`FromWuttaToVersionBase`.
+
+ Its (target)
+ :attr:`~wuttasync.importing.base.Importer.model_class` will be
+ set to the **version** model.
+
+ Its
+ :attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class`
+ will be set to the **normal** model.
+
+ :param model_class: A (normal, not version) data model class.
+
+ :param name: The "model name" for the importer. New class
+ name will be based on this, so e.g. ``Widget`` model name
+ becomes ``WidgetImporter`` class name.
+
+ :returns: The new class, or ``None``
+ """
+ import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
+
+ try:
+ version_class = continuum.version_class(model_class)
+ except continuum.exc.ClassNotVersioned:
+ return None
+
+ return type(
+ f"{name}Importer",
+ (FromWuttaToVersionBase,),
+ {
+ "source_model_class": model_class,
+ "model_class": version_class,
+ "default_keys": list(get_primary_keys(model_class)),
+ },
+ )
+
+
+class FromWuttaToVersionBase(FromWuttaMirror, ToWutta):
+ """
+ Base importer class for Wutta -> Versions.
+
+ This imports from
+ :class:`~wuttasync.importing.wutta.FromWuttaMirror` and
+ :class:`~wuttasync.importing.model.ToWutta`.
+
+ The import handler will dynamically generate importers using this
+ base class; see
+ :meth:`~FromWuttaToVersions.make_importer_factory()`.
+ """
+
+ continuum_txn = None
+ """
+ Reference to the handler's attribute of the same name:
+ :attr:`~FromWuttaToVersions.continuum_txn`
+
+ This is the SQLAlchemy-Continuum ``transaction`` record, to which
+ any new version records will associate (if needed).
+
+ This transaction will track the effective user responsible for
+ the change(s), their client IP, and timestamp.
+ """
+
+ def get_simple_fields(self): # pylint: disable=empty-docstring
+ """ """
+ fields = super().get_simple_fields()
+ unwanted = ["transaction_id", "operation_type", "end_transaction_id"]
+ fields = [field for field in fields if field not in unwanted]
+ return fields
+
+ def get_target_query(self, source_data=None):
+ """
+ This modifies the normal query to ensure we only get the
+ "latest valid" version for each record, for comparison to
+ source.
+
+ .. note::
+
+ In some cases, it still may be possible for multiple
+ "latest" versions to match for a given record. This means
+ inconsistent data; a warning should be logged if so, and
+ you must track it down...
+
+ See also docs for parent method:
+ :meth:`~wuttasync.importing.base.ToSqlalchemy.get_target_query()`
+ """
+ import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
+
+ # pylint: disable=singleton-comparison
+ return (
+ self.target_session.query(self.model_class)
+ .filter(self.model_class.end_transaction_id == None)
+ .filter(self.model_class.operation_type != continuum.Operation.DELETE)
+ )
+
+ def normalize_target_object(self, obj): # pylint: disable=empty-docstring
+ """ """
+ data = super().normalize_target_object(obj)
+
+ # we want to add the original version object to normalized
+ # data, so we can access it later for updating if needed. but
+ # this method is called for *both* sides (source+target) since
+ # this is a "mirrored" importer. so we must check the type
+ # and only cache true versions, ignore "normal" objects.
+ if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type
+ obj, self.model_class
+ ):
+ data["_version"] = obj
+
+ return data
+
+ def make_version( # pylint: disable=missing-function-docstring
+ self, source_data, operation_type
+ ):
+ key = self.get_record_key(source_data)
+ with self.target_session.no_autoflush:
+ version = self.make_empty_object(key)
+ self.populate(version, source_data)
+ version.transaction = self.continuum_txn
+ version.operation_type = operation_type
+ self.target_session.add(version)
+ return version
+
+ def populate(self, obj, data): # pylint: disable=missing-function-docstring
+ keys = self.get_keys()
+ for field in self.get_simple_fields():
+ if field not in keys and field in data and field in self.fields:
+ setattr(obj, field, data[field])
+
+ def create_target_object(self, key, source_data): # pylint: disable=empty-docstring
+ """ """
+ import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
+
+ return self.make_version(source_data, continuum.Operation.INSERT)
+
+ def update_target_object( # pylint: disable=empty-docstring
+ self, obj, source_data, target_data=None
+ ):
+ """ """
+ import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
+
+ # when we "update" it always involves making a *new* version
+ # record. but that requires actually updating the "previous"
+ # version to indicate the new version's transaction.
+ prev_version = target_data.pop("_version")
+ prev_version.end_transaction_id = self.continuum_txn.id
+
+ return self.make_version(source_data, continuum.Operation.UPDATE)
+
+ def delete_target_object(self, obj): # pylint: disable=empty-docstring
+ """ """
+ import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
+
+ # nb. `obj` here is the existing/old version record; we update
+ # it to indicate the new version's transaction.
+ obj.end_transaction_id = self.continuum_txn.id
+
+ # add new "DELETE" version record. values should be the same as
+ # for "previous" (existing/old) version.
+ source_data = self.normalize_target_object(obj)
+ return self.make_version(source_data, continuum.Operation.DELETE)
diff --git a/src/wuttasync/importing/wutta.py b/src/wuttasync/importing/wutta.py
index 9de4822..882f7df 100644
--- a/src/wuttasync/importing/wutta.py
+++ b/src/wuttasync/importing/wutta.py
@@ -21,37 +21,13 @@
#
################################################################################
"""
-Wutta ⇄ Wutta import/export
+Wutta → Wutta import/export
"""
-from .handlers import ToSqlalchemyHandler
+from .base import FromSqlalchemyMirror
-class ToWuttaHandler(ToSqlalchemyHandler):
+class FromWuttaMirror(FromSqlalchemyMirror): # pylint: disable=abstract-method
"""
- Handler for import/export which targets Wutta ORM (:term:`app
- database`).
+ Base class for Wutta -> Wutta data importers.
"""
-
- target_key = "wutta"
- "" # nb. suppress docs
-
- def get_target_title(self): # pylint: disable=empty-docstring
- """ """
- # nb. we override parent to use app title as default
- if hasattr(self, "target_title"):
- return self.target_title
- if hasattr(self, "generic_target_title"):
- return self.generic_target_title
- return self.app.get_title()
-
- def make_target_session(self):
- """
- Call
- :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
- and return the result.
-
- :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
- instance.
- """
- return self.app.make_session()
diff --git a/tests/cli/test_import_versions.py b/tests/cli/test_import_versions.py
new file mode 100644
index 0000000..ea1617d
--- /dev/null
+++ b/tests/cli/test_import_versions.py
@@ -0,0 +1,22 @@
+# -*- coding: utf-8; -*-
+
+from unittest import TestCase
+from unittest.mock import MagicMock, patch
+
+from wuttasync.cli import import_versions as mod, ImportCommandHandler
+
+
+class TestImportCsv(TestCase):
+
+ def test_basic(self):
+ params = {
+ "models": [],
+ "create": True,
+ "update": True,
+ "delete": False,
+ "dry_run": True,
+ }
+ ctx = MagicMock(params=params)
+ with patch.object(ImportCommandHandler, "run") as run:
+ mod.import_versions(ctx)
+ run.assert_called_once_with(params)
diff --git a/tests/importing/test_base.py b/tests/importing/test_base.py
index 08c37a2..c920ed6 100644
--- a/tests/importing/test_base.py
+++ b/tests/importing/test_base.py
@@ -2,6 +2,8 @@
from unittest.mock import patch
+from sqlalchemy import orm
+
from wuttjamaican.testing import DataTestCase
from wuttasync.importing import base as mod, ImportHandler, Orientation
@@ -78,13 +80,31 @@ class TestImporter(DataTestCase):
def test_get_keys(self):
model = self.app.model
+
+ # nb. get_keys() will cache the return value, so must
+ # re-create importer for each test
+
+ # keys inspected from model by default
imp = self.make_importer(model_class=model.Setting)
self.assertEqual(imp.get_keys(), ["name"])
- with patch.multiple(imp, create=True, key="value"):
- self.assertEqual(imp.get_keys(), ["value"])
- with patch.multiple(imp, create=True, keys=["foo", "bar"]):
+ imp = self.make_importer(model_class=model.User)
+ self.assertEqual(imp.get_keys(), ["uuid"])
+
+ # class may define 'keys'
+ imp = self.make_importer(model_class=model.User)
+ with patch.object(imp, "keys", new=["foo", "bar"], create=True):
self.assertEqual(imp.get_keys(), ["foo", "bar"])
+ # class may define 'key'
+ imp = self.make_importer(model_class=model.User)
+ with patch.object(imp, "key", new="whatever", create=True):
+ self.assertEqual(imp.get_keys(), ["whatever"])
+
+ # class may define 'default_keys'
+ imp = self.make_importer(model_class=model.User)
+ with patch.object(imp, "default_keys", new=["baz", "foo"]):
+ self.assertEqual(imp.get_keys(), ["baz", "foo"])
+
def test_process_data(self):
model = self.app.model
imp = self.make_importer(
@@ -651,6 +671,105 @@ class TestFromFile(DataTestCase):
close.assert_called_once_with()
+class TestQueryWrapper(DataTestCase):
+
+ def test_basic(self):
+ model = self.app.model
+
+ p1 = model.Person(full_name="John Doe")
+ self.session.add(p1)
+ p2 = model.Person(full_name="Jane Doe")
+ self.session.add(p2)
+ self.session.commit()
+
+ # cannot get count via len(query), must use query.count()
+ query = self.session.query(model.Person)
+ self.assertEqual(query.count(), 2)
+ self.assertRaises(TypeError, len, query)
+
+ # but can use len(wrapper)
+ wrapper = mod.QueryWrapper(query)
+ self.assertEqual(len(wrapper), 2)
+
+ # iter(wrapper) should work too
+ people = [p for p in wrapper]
+ self.assertEqual(people, [p1, p2])
+ people = [p for p in iter(wrapper)]
+ self.assertEqual(people, [p1, p2])
+ people = [p for p in list(wrapper)]
+ self.assertEqual(people, [p1, p2])
+
+
+class TestFromSqlalchemy(DataTestCase):
+
+ def setUp(self):
+ self.setup_db()
+ self.handler = ImportHandler(self.config)
+
+ def make_importer(self, **kwargs):
+ kwargs.setdefault("handler", self.handler)
+ return mod.FromSqlalchemy(self.config, **kwargs)
+
+ def test_get_source_query(self):
+ model = self.app.model
+ imp = self.make_importer(
+ source_model_class=model.Upgrade, source_session=self.session
+ )
+ query = imp.get_source_query()
+ self.assertIsInstance(query, orm.Query)
+ self.assertEqual(len(query.selectable.froms), 1)
+ table = query.selectable.froms[0]
+ self.assertEqual(table.name, "upgrade")
+
+ def test_get_source_objects(self):
+ model = self.app.model
+
+ user1 = model.User(username="fred")
+ self.session.add(user1)
+ user2 = model.User(username="bettie")
+ self.session.add(user2)
+ self.session.commit()
+
+ imp = self.make_importer(
+ source_model_class=model.User, source_session=self.session
+ )
+ result = imp.get_source_objects()
+ self.assertIsInstance(result, mod.QueryWrapper)
+ self.assertEqual(len(result), 2)
+ self.assertEqual(list(result), [user1, user2])
+
+
+class TestFromSqlalchemyMirror(DataTestCase):
+
+ def setUp(self):
+ self.setup_db()
+ self.handler = ImportHandler(self.config)
+
+ def make_importer(self, **kwargs):
+ kwargs.setdefault("handler", self.handler)
+ return mod.FromSqlalchemyMirror(self.config, **kwargs)
+
+ def test_source_model_class(self):
+ model = self.app.model
+
+ # source_model_class will mirror model_class
+ imp = self.make_importer(model_class=model.Upgrade)
+ self.assertIs(imp.model_class, model.Upgrade)
+ self.assertIs(imp.source_model_class, model.Upgrade)
+
+ def test_normalize_source_object(self):
+ model = self.app.model
+ imp = self.make_importer(model_class=model.Upgrade)
+ upgrade = model.Upgrade()
+
+ # normalize_source_object() should invoke normalize_target_object()
+ with patch.object(imp, "normalize_target_object") as normalize_target_object:
+ normalize_target_object.return_value = 42
+ result = imp.normalize_source_object(upgrade)
+ self.assertEqual(result, 42)
+ normalize_target_object.assert_called_once_with(upgrade)
+
+
class TestToSqlalchemy(DataTestCase):
def setUp(self):
diff --git a/tests/importing/test_handlers.py b/tests/importing/test_handlers.py
index 9bd0157..a6df032 100644
--- a/tests/importing/test_handlers.py
+++ b/tests/importing/test_handlers.py
@@ -213,6 +213,97 @@ class TestFromFileHandler(DataTestCase):
process_data.assert_called_once_with(input_file_dir=self.tempdir)
+class TestFromSqlalchemyHandler(DataTestCase):
+
+ def make_handler(self, **kwargs):
+ return mod.FromSqlalchemyHandler(self.config, **kwargs)
+
+ def test_make_source_session(self):
+ handler = self.make_handler()
+ self.assertRaises(NotImplementedError, handler.make_source_session)
+
+ def test_begin_source_transaction(self):
+ handler = self.make_handler()
+ self.assertIsNone(handler.source_session)
+ with patch.object(handler, "make_source_session", return_value=self.session):
+ handler.begin_source_transaction()
+ self.assertIs(handler.source_session, self.session)
+
+ def test_commit_source_transaction(self):
+ model = self.app.model
+ handler = self.make_handler()
+ handler.source_session = self.session
+ self.assertEqual(self.session.query(model.User).count(), 0)
+
+ # nb. do not commit this yet
+ user = model.User(username="fred")
+ self.session.add(user)
+
+ self.assertTrue(self.session.in_transaction())
+ self.assertIn(user, self.session)
+ handler.commit_source_transaction()
+ self.assertIsNone(handler.source_session)
+ self.assertFalse(self.session.in_transaction())
+ self.assertNotIn(user, self.session) # hm, surprising?
+ self.assertEqual(self.session.query(model.User).count(), 1)
+
+ def test_rollback_source_transaction(self):
+ model = self.app.model
+ handler = self.make_handler()
+ handler.source_session = self.session
+ self.assertEqual(self.session.query(model.User).count(), 0)
+
+ # nb. do not commit this yet
+ user = model.User(username="fred")
+ self.session.add(user)
+
+ self.assertTrue(self.session.in_transaction())
+ self.assertIn(user, self.session)
+ handler.rollback_source_transaction()
+ self.assertIsNone(handler.source_session)
+ self.assertFalse(self.session.in_transaction())
+ self.assertNotIn(user, self.session)
+ self.assertEqual(self.session.query(model.User).count(), 0)
+
+ def test_get_importer_kwargs(self):
+ handler = self.make_handler()
+ handler.source_session = self.session
+ kw = handler.get_importer_kwargs("User")
+ self.assertIn("source_session", kw)
+ self.assertIs(kw["source_session"], self.session)
+
+
+class TestFromWuttaHandler(DataTestCase):
+
+ def make_handler(self, **kwargs):
+ return mod.FromWuttaHandler(self.config, **kwargs)
+
+ def test_get_source_title(self):
+ handler = self.make_handler()
+
+ # uses app title by default
+ self.config.setdefault("wutta.app_title", "What About This")
+ self.assertEqual(handler.get_source_title(), "What About This")
+
+ # or generic default if present
+ handler.generic_source_title = "WHATABOUTTHIS"
+ self.assertEqual(handler.get_source_title(), "WHATABOUTTHIS")
+
+ # but prefer specific title if present
+ handler.source_title = "what_about_this"
+ self.assertEqual(handler.get_source_title(), "what_about_this")
+
+ def test_make_source_session(self):
+ handler = self.make_handler()
+
+ # makes "new" (mocked in our case) app session
+ with patch.object(self.app, "make_session") as make_session:
+ make_session.return_value = self.session
+ session = handler.make_source_session()
+ make_session.assert_called_once_with()
+ self.assertIs(session, self.session)
+
+
class TestToSqlalchemyHandler(DataTestCase):
def make_handler(self, **kwargs):
@@ -256,3 +347,34 @@ class TestToSqlalchemyHandler(DataTestCase):
kw = handler.get_importer_kwargs("Setting")
self.assertIn("target_session", kw)
self.assertIs(kw["target_session"], self.session)
+
+
+class TestToWuttaHandler(DataTestCase):
+
+ def make_handler(self, **kwargs):
+ return mod.ToWuttaHandler(self.config, **kwargs)
+
+ def test_get_target_title(self):
+ handler = self.make_handler()
+
+ # uses app title by default
+ self.config.setdefault("wutta.app_title", "What About This")
+ self.assertEqual(handler.get_target_title(), "What About This")
+
+ # or generic default if present
+ handler.generic_target_title = "WHATABOUTTHIS"
+ self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS")
+
+ # but prefer specific title if present
+ handler.target_title = "what_about_this"
+ self.assertEqual(handler.get_target_title(), "what_about_this")
+
+ def test_make_target_session(self):
+ handler = self.make_handler()
+
+ # makes "new" (mocked in our case) app session
+ with patch.object(self.app, "make_session") as make_session:
+ make_session.return_value = self.session
+ session = handler.make_target_session()
+ make_session.assert_called_once_with()
+ self.assertIs(session, self.session)
diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py
new file mode 100644
index 0000000..2067f93
--- /dev/null
+++ b/tests/importing/test_versions.py
@@ -0,0 +1,247 @@
+# -*- coding: utf-8; -*-
+
+from sqlalchemy import orm
+import sqlalchemy_continuum as continuum
+
+from wuttjamaican.util import make_true_uuid
+from wutta_continuum.testing import VersionTestCase
+
+from wuttasync.importing import versions as mod, Importer
+
+
+class TestFromWuttaToVersions(VersionTestCase):
+
+ def make_handler(self, **kwargs):
+ return mod.FromWuttaToVersions(self.config, **kwargs)
+
+ def test_begin_target_transaction(self):
+ model = self.app.model
+ txncls = continuum.transaction_class(model.User)
+
+ handler = self.make_handler()
+ self.assertIsNone(handler.continuum_uow)
+ self.assertIsNone(handler.continuum_txn)
+
+ handler.begin_target_transaction()
+ self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork)
+ self.assertIsInstance(handler.continuum_txn, txncls)
+
+ def test_get_importer_kwargs(self):
+ handler = self.make_handler()
+ handler.begin_target_transaction()
+
+ kw = handler.get_importer_kwargs("User")
+ self.assertIn("continuum_txn", kw)
+ self.assertIs(kw["continuum_txn"], handler.continuum_txn)
+
+ def test_make_importer_factory(self):
+ model = self.app.model
+ handler = self.make_handler()
+
+ # versioned class
+ factory = handler.make_importer_factory(model.User, "User")
+ self.assertTrue(issubclass(factory, mod.FromWuttaToVersionBase))
+ self.assertIs(factory.source_model_class, model.User)
+ self.assertIs(factory.model_class, continuum.version_class(model.User))
+
+ # non-versioned
+ factory = handler.make_importer_factory(model.Upgrade, "Upgrade")
+ self.assertIsNone(factory)
+
+ def test_define_importers(self):
+ handler = self.make_handler()
+
+ importers = handler.define_importers()
+ self.assertIn("User", importers)
+ self.assertIn("Person", importers)
+ self.assertNotIn("Upgrade", importers)
+
+
+class UserImporter(mod.FromWuttaToVersionBase):
+
+ @property
+ def model_class(self):
+ model = self.app.model
+ return model.User
+
+
+class TestFromWuttaToVersionBase(VersionTestCase):
+
+ def make_importer(self, model_class=None, **kwargs):
+ imp = mod.FromWuttaToVersionBase(self.config, **kwargs)
+ if model_class:
+ imp.model_class = model_class
+ return imp
+
+ def test_get_simple_fields(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+
+ # first confirm what a "normal" importer would do
+ imp = Importer(self.config, model_class=vercls)
+ fields = imp.get_simple_fields()
+ self.assertIn("username", fields)
+ self.assertIn("person_uuid", fields)
+ self.assertIn("transaction_id", fields)
+ self.assertIn("operation_type", fields)
+ self.assertIn("end_transaction_id", fields)
+
+ # now test what the "version" importer does
+ imp = self.make_importer(model_class=vercls)
+ fields = imp.get_simple_fields()
+ self.assertIn("username", fields)
+ self.assertIn("person_uuid", fields)
+ self.assertNotIn("transaction_id", fields)
+ self.assertNotIn("operation_type", fields)
+ self.assertNotIn("end_transaction_id", fields)
+
+ def test_get_target_query(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+ imp = self.make_importer(model_class=vercls, target_session=self.session)
+
+ # TODO: not sure what else to test here..
+ query = imp.get_target_query()
+ self.assertIsInstance(query, orm.Query)
+
+ def test_normalize_target_object(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+ imp = self.make_importer(model_class=vercls)
+
+ user = model.User(username="fred")
+ self.session.add(user)
+ self.session.commit()
+ version = user.versions[0]
+
+ # version object should be embedded in data dict
+ data = imp.normalize_target_object(version)
+ self.assertIsInstance(data, dict)
+ self.assertIn("_version", data)
+ self.assertIs(data["_version"], version)
+
+ # but normal object is not embedded
+ data = imp.normalize_target_object(user)
+ self.assertIsInstance(data, dict)
+ self.assertNotIn("_version", data)
+
+ def test_make_version(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+
+ user = model.User(username="fred")
+ self.session.add(user)
+ self.session.commit()
+
+ handler = mod.FromWuttaToVersions(self.config)
+ handler.begin_target_transaction()
+ handler.target_session.close()
+ handler.target_session = self.session
+
+ imp = self.make_importer(
+ model_class=vercls,
+ fields=["uuid", "username"],
+ keys=("uuid",),
+ target_session=self.session,
+ continuum_txn=handler.continuum_txn,
+ )
+
+ data = {"uuid": user.uuid, "username": "freddie"}
+ version = imp.make_version(data, continuum.Operation.UPDATE)
+ self.assertIsInstance(version, vercls)
+ self.assertEqual(version.uuid, user.uuid)
+ self.assertEqual(version.username, "freddie")
+ self.assertIn(version, self.session)
+ self.assertIs(version.transaction, imp.continuum_txn)
+ self.assertEqual(version.operation_type, continuum.Operation.UPDATE)
+
+ def test_create_target_object(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+
+ handler = mod.FromWuttaToVersions(self.config)
+ handler.begin_target_transaction()
+ handler.target_session.close()
+ handler.target_session = self.session
+
+ imp = self.make_importer(
+ model_class=vercls,
+ fields=["uuid", "username"],
+ keys=("uuid",),
+ target_session=self.session,
+ continuum_txn=handler.continuum_txn,
+ )
+
+ source_data = {"uuid": make_true_uuid(), "username": "bettie"}
+ self.assertEqual(self.session.query(vercls).count(), 0)
+ version = imp.create_target_object((source_data["uuid"], 1), source_data)
+ self.assertEqual(self.session.query(vercls).count(), 1)
+ self.assertEqual(version.transaction_id, imp.continuum_txn.id)
+ self.assertEqual(version.operation_type, continuum.Operation.INSERT)
+ self.assertIsNone(version.end_transaction_id)
+
+ def test_update_target_object(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+
+ user = model.User(username="fred")
+ self.session.add(user)
+ self.session.commit()
+ version1 = user.versions[0]
+
+ handler = mod.FromWuttaToVersions(self.config)
+ handler.begin_target_transaction()
+ handler.target_session.close()
+ handler.target_session = self.session
+
+ imp = self.make_importer(
+ model_class=vercls,
+ fields=["uuid", "username"],
+ keys=("uuid",),
+ target_session=self.session,
+ continuum_txn=handler.continuum_txn,
+ )
+
+ source_data = {"uuid": user.uuid, "username": "freddie"}
+ target_data = imp.normalize_target_object(version1)
+ self.assertEqual(self.session.query(vercls).count(), 1)
+ self.assertIsNone(version1.end_transaction_id)
+ version2 = imp.update_target_object(
+ version1, source_data, target_data=target_data
+ )
+ self.assertEqual(self.session.query(vercls).count(), 2)
+ self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id)
+ self.assertEqual(version2.transaction_id, imp.continuum_txn.id)
+ self.assertEqual(version2.operation_type, continuum.Operation.UPDATE)
+ self.assertIsNone(version2.end_transaction_id)
+
+ def test_delete_target_object(self):
+ model = self.app.model
+ vercls = continuum.version_class(model.User)
+
+ user = model.User(username="fred")
+ self.session.add(user)
+ self.session.commit()
+ version1 = user.versions[0]
+
+ handler = mod.FromWuttaToVersions(self.config)
+ handler.begin_target_transaction()
+ handler.target_session.close()
+ handler.target_session = self.session
+
+ imp = self.make_importer(
+ model_class=vercls,
+ fields=["uuid", "username"],
+ keys=("uuid",),
+ target_session=self.session,
+ continuum_txn=handler.continuum_txn,
+ )
+
+ self.assertEqual(self.session.query(vercls).count(), 1)
+ self.assertIsNone(version1.end_transaction_id)
+ version2 = imp.delete_target_object(version1)
+ self.assertEqual(self.session.query(vercls).count(), 2)
+ self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id)
+ self.assertEqual(version2.transaction_id, imp.continuum_txn.id)
+ self.assertEqual(version2.operation_type, continuum.Operation.DELETE)
+ self.assertIsNone(version2.end_transaction_id)
diff --git a/tests/importing/test_wutta.py b/tests/importing/test_wutta.py
index 4d6fdd2..1533605 100644
--- a/tests/importing/test_wutta.py
+++ b/tests/importing/test_wutta.py
@@ -1,38 +1,3 @@
# -*- coding: utf-8; -*-
-from unittest.mock import patch
-
-from wuttjamaican.testing import DataTestCase
-
from wuttasync.importing import wutta as mod
-
-
-class TestToWuttaHandler(DataTestCase):
-
- def make_handler(self, **kwargs):
- return mod.ToWuttaHandler(self.config, **kwargs)
-
- def test_get_target_title(self):
- handler = self.make_handler()
-
- # uses app title by default
- self.config.setdefault("wutta.app_title", "What About This")
- self.assertEqual(handler.get_target_title(), "What About This")
-
- # or generic default if present
- handler.generic_target_title = "WHATABOUTTHIS"
- self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS")
-
- # but prefer specific title if present
- handler.target_title = "what_about_this"
- self.assertEqual(handler.get_target_title(), "what_about_this")
-
- def test_make_target_session(self):
- handler = self.make_handler()
-
- # makes "new" (mocked in our case) app session
- with patch.object(self.app, "make_session") as make_session:
- make_session.return_value = self.session
- session = handler.make_target_session()
- make_session.assert_called_once_with()
- self.assertIs(session, self.session)