feat: add warnings mode for import/export handlers, commands

can now specify `--warn` for import/export CLI, to get diff email when
changes occur.

this also adds `get_import_handler()` and friends, via app provider.

also declare email settings for the 2 existing importers

docs/api/wuttasync.app.rst

@@ -0,0 +1,6 @@
+
+``wuttasync.app``
+=================
+
+.. automodule:: wuttasync.app
+   :members:

docs/api/wuttasync.emails.rst

@@ -0,0 +1,6 @@
+
+``wuttasync.emails``
+====================
+
+.. automodule:: wuttasync.emails
+   :members:

docs/api/wuttasync.testing.rst

@@ -0,0 +1,6 @@
+
+``wuttasync.testing``
+=====================
+
+.. automodule:: wuttasync.testing
+   :members:

docs/glossary.rst

@@ -6,6 +6,26 @@ Glossary
 .. glossary::
    :sorted:
 
+   import/export key
+     Unique key representing a particular type of import/export job,
+     i.e. the source/target and orientation (import vs. export).
+
+     For instance "Wutta → CSV export" uses the key:
+     ``export.to_csv.from_wutta``
+
+     More than one :term:`import handler` can share a key, e.g. one
+     may subclass another and inherit the key.
+
+     However only one handler is "designated" for a given key; it will
+     be used by default for running those jobs.
+
+     This key is used for lookup in
+     :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`.
+
+     See also
+     :meth:`~wuttasync.importing.handlers.ImportHandler.get_key()`
+     method on the import/export handler.
+
    import handler
      This a type of :term:`handler` which is responsible for a
      particular set of data import/export task(s).

docs/index.rst

@@ -67,13 +67,15 @@ cf. :doc:`rattail-manual:data/sync/index`.
 
 .. toctree::
    :maxdepth: 1
-   :caption: API
+   :caption: Package API
 
    api/wuttasync
+   api/wuttasync.app
    api/wuttasync.cli
    api/wuttasync.cli.base
    api/wuttasync.cli.import_csv
    api/wuttasync.cli.import_versions
+   api/wuttasync.emails
    api/wuttasync.importing
    api/wuttasync.importing.base
    api/wuttasync.importing.csv
@@ -81,4 +83,5 @@ cf. :doc:`rattail-manual:data/sync/index`.
    api/wuttasync.importing.model
    api/wuttasync.importing.versions
    api/wuttasync.importing.wutta
+   api/wuttasync.testing
    api/wuttasync.util

pyproject.toml

@@ -26,6 +26,7 @@ classifiers = [
 ]
 requires-python = ">= 3.8"
 dependencies = [
+        "humanize",
         "makefun",
         "SQLAlchemy-Utils",
         "WuttJamaican[db]>=0.16.2",
@@ -37,6 +38,13 @@ docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
 tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"]
 
 
+[project.entry-points."wutta.app.providers"]
+wuttasync = "wuttasync.app:WuttaSyncAppProvider"
+
+[project.entry-points."wuttasync.importing"]
+"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions"
+"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta"
+
 [project.entry-points."wutta.typer_imports"]
 wuttasync = "wuttasync.cli"
 

src/wuttasync/app.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  WuttaSync -- Wutta Framework for data import/export and real-time sync
+#  Copyright © 2024-2025 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+App handler supplement for WuttaSync
+"""
+
+from collections import OrderedDict
+
+from wuttjamaican.app import AppProvider
+from wuttjamaican.util import load_entry_points
+
+
+class WuttaSyncAppProvider(AppProvider):
+    """
+    The :term:`app provider` for WuttaSync.
+
+    This adds some methods to the :term:`app handler`, which are
+    specific to import/export.
+
+    It also declares some :term:`email modules <email module>` and
+    :term:`email templates <email template>` for the app.
+
+    We have two concerns when doing lookups etc. for import/export
+    handlers:
+
+    * which handlers are *available* - i.e. they exist and are
+      discoverable
+    * which handlers are *designated* - only one designated handler
+      per key
+
+    All "available" handlers will have a key, but some keys may be
+    referenced by multiple handlers.  For each key, only one handler
+    can be "designated" - there is a default, but config can override.
+    """
+
+    email_modules = ["wuttasync.emails"]
+    email_templates = ["wuttasync:email-templates"]
+
+    def get_all_import_handlers(self):
+        """
+        Returns *all* :term:`import/export handler <import handler>`
+        *classes* which are known to exist, i.e.  are discoverable.
+
+        See also :meth:`get_import_handler()` and
+        :meth:`get_designated_import_handlers()`.
+
+        The discovery process is as follows:
+
+        * load handlers from registered entry points
+        * check config for designated handlers
+
+        Checking for designated handler config is not a reliable way
+        to discover handlers, but it's done just in case any new ones
+        might be found.
+
+        Registration via entry points is the only way to ensure a
+        handler is discoverable.  The entry point group name is always
+        ``wuttasync.importing`` regardless of :term:`app name`;
+        entries are like ``"handler_key" = "handler_spec"``.  For
+        example:
+
+        .. code-block:: toml
+
+           [project.entry-points."wuttasync.importing"]
+           "export.to_csv.from_poser" = "poser.exporting.csv:FromPoserToCsv"
+           "import.to_poser.from_csv" = "poser.importing.csv:FromCsvToPoser"
+
+        :returns: List of all import/export handler classes
+        """
+        # first load all "registered" Handler classes
+        factories = load_entry_points("wuttasync.importing", ignore_errors=True)
+
+        # organize registered classes by spec
+        specs = {factory.get_spec(): factory for factory in factories.values()}
+
+        # many handlers may not be registered per se, but may be
+        # designated via config.  so try to include those too
+        for factory in factories.values():
+            spec = self.get_designated_import_handler_spec(factory.get_key())
+            if spec and spec not in specs:
+                specs[spec] = self.app.load_object(spec)
+
+        # flatten back to simple list of classes
+        factories = list(specs.values())
+        return factories
+
+    def get_designated_import_handler_spec(self, key, require=False):
+        """
+        Returns the designated import/export handler :term:`spec`
+        string for the given type key.
+
+        This just checks config for the designated handler, using the
+        ``wuttasync.importing`` prefix regardless of :term:`app name`.
+        For instance:
+
+        .. code-block:: ini
+
+           [wuttasync.importing]
+           export.to_csv.from_poser.handler = poser.exporting.csv:FromPoserToCsv
+           import.to_poser.from_csv.handler = poser.importing.csv:FromCsvToPoser
+
+        See also :meth:`get_designated_import_handlers()` and
+        :meth:`get_import_handler()`.
+
+        :param key: Unique key indicating the type of import/export
+           handler.
+
+        :param require: Flag indicating whether an error should be raised if no
+           handler is found.
+
+        :returns: Spec string for the designated handler.  If none is
+           configured, then ``None`` is returned *unless* the
+           ``require`` param is true, in which case an error is
+           raised.
+        """
+        spec = self.config.get(f"wuttasync.importing.{key}.handler")
+        if spec:
+            return spec
+
+        spec = self.config.get(f"wuttasync.importing.{key}.default_handler")
+        if spec:
+            return spec
+
+        if require:
+            raise ValueError(f"Cannot locate import handler spec for key: {key}")
+        return None
+
+    def get_designated_import_handlers(self):
+        """
+        Returns all *designated* import/export handler *instances*.
+
+        Each import/export handler has a "key" which indicates the
+        "type" of import/export job it performs.  For instance the CSV
+        → Wutta import has the key: ``import.to_wutta.from_csv``
+
+        More than one handler can be defined for that key; however
+        only one such handler will be "designated" for each key.
+
+        This method first loads *all* available import handlers, then
+        organizes them by key, and tries to determine which handler
+        should be designated for each key.
+
+        See also :meth:`get_all_import_handlers()` and
+        :meth:`get_designated_import_handler_spec()`.
+
+        :returns: List of designated import/export handler instances
+        """
+        grouped = OrderedDict()
+        for factory in self.get_all_import_handlers():
+            key = factory.get_key()
+            grouped.setdefault(key, []).append(factory)
+
+        def find_designated(key, group):
+            spec = self.get_designated_import_handler_spec(key)
+            if spec:
+                for factory in group:
+                    if factory.get_spec() == spec:
+                        return factory
+            if len(group) == 1:
+                return group[0]
+            return None
+
+        designated = []
+        for key, group in grouped.items():
+            factory = find_designated(key, group)
+            if factory:
+                handler = factory(self.config)
+                designated.append(handler)
+
+        return designated
+
+    def get_import_handler(self, key, require=False, **kwargs):
+        """
+        Returns the designated :term:`import/export handler <import
+        handler>` instance for the given :term:`import/export key`.
+
+        See also :meth:`get_all_import_handlers()` and
+        :meth:`get_designated_import_handlers()`.
+
+        :param key: Key indicating the type of import/export handler,
+           e.g. ``"import.to_wutta.from_csv"``
+
+        :param require: Set this to true if you want an error raised
+           when no handler is found.
+
+        :returns: The import/export handler instance.  If no handler
+           is found, then ``None`` is returned, unless ``require``
+           param is true, in which case error is raised.
+        """
+        # first try to fetch the handler per designated spec
+        spec = self.get_designated_import_handler_spec(key, **kwargs)
+        if spec:
+            factory = self.app.load_object(spec)
+            return factory(self.config)
+
+        # nothing was designated, so leverage logic which already
+        # sorts out which handler is "designated" for given key
+        designated = self.get_designated_import_handlers()
+        for handler in designated:
+            if handler.get_key() == key:
+                return handler
+
+        if require:
+            raise ValueError(f"Cannot locate import handler for key: {key}")
+        return None

src/wuttasync/cli/base.py

@@ -127,15 +127,16 @@ class ImportCommandHandler(GenericHandler):
 
         This is what happens when command line has ``--list-models``.
         """
-        sys.stdout.write("ALL MODELS:\n")
+        sys.stdout.write("\nALL MODELS:\n")
         sys.stdout.write("==============================\n")
         for key in self.import_handler.importers:
             sys.stdout.write(key)
             sys.stdout.write("\n")
         sys.stdout.write("==============================\n")
+        sys.stdout.write(f"for {self.import_handler.get_title()}\n\n")
 
 
-def import_command_template(  # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments
+def import_command_template(  # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments,too-many-locals
     models: Annotated[
         Optional[List[str]],
         typer.Argument(
@@ -217,6 +218,27 @@ def import_command_template(  # pylint: disable=unused-argument,too-many-argumen
             help="Max number of *any* target record changes which may occur (per model)."
         ),
     ] = None,
+    warnings: Annotated[
+        bool,
+        typer.Option(
+            "--warn",
+            "-W",
+            help="Expect no changes; warn (email the diff) if any occur.",
+        ),
+    ] = False,
+    warnings_recipients: Annotated[
+        str,
+        typer.Option(
+            "--recip", help="Override the recipient(s) for diff warning email."
+        ),
+    ] = None,
+    warnings_max_diffs: Annotated[
+        int,
+        typer.Option(
+            "--max-diffs",
+            help="Max number of record diffs to show (per model) in warning email.",
+        ),
+    ] = 15,
     dry_run: Annotated[
         bool,
         typer.Option(

src/wuttasync/email-templates/import_export_warning.html.mako

@@ -0,0 +1,88 @@
+## -*- coding: utf-8; -*-
+<html>
+  <body>
+    <h3>Diff warning for ${title} (${handler.actioning})</h3>
+
+    <p style="font-style: italic;">
+      % if dry_run:
+          <span style="font-weight: bold;">DRY RUN</span>
+          - these changes have not yet happened
+      % else:
+          <span style="font-weight: bold;">LIVE RUN</span>
+          - these changes already happened
+      % endif
+    </p>
+
+    <ul>
+      % for model, (created, updated, deleted) in changes.items():
+          <li>
+            <a href="#${model}">${model}</a> -
+            ${app.render_quantity(len(created))} created;
+            ${app.render_quantity(len(updated))} updated;
+            ${app.render_quantity(len(deleted))} deleted
+          </li>
+      % endfor
+    </ul>
+
+    <p>
+      <span style="font-weight: bold;">COMMAND:</span>
+      &nbsp;
+      <code>${argv}</code>
+    </p>
+
+    <p>
+      <span style="font-weight: bold;">RUNTIME:</span>
+      &nbsp;
+      ${runtime} (${runtime_display})
+    </p>
+
+    % for model, (created, updated, deleted) in changes.items():
+
+        <br />
+        <h4>
+          <a name="${model}">${model}</a> -
+          ${app.render_quantity(len(created))} created;
+          ${app.render_quantity(len(updated))} updated;
+          ${app.render_quantity(len(deleted))} deleted
+        </h4>
+
+        <div style="padding-left: 2rem;">
+
+          % for obj, source_data in created[:max_diffs]:
+              <h5>${model} <em>created</em> in ${target_title}: ${obj}</h5>
+              <% diff = make_diff({}, source_data, nature="create") %>
+              <div style="padding-left: 2rem;">
+                ${diff.render_html()}
+              </div>
+          % endfor
+          % if len(created) > max_diffs:
+              <h5>${model} - ${app.render_quantity(len(created) - max_diffs)} more records <em>created</em> in ${target_title} - not shown here</h5>
+          % endif
+
+          % for obj, source_data, target_data in updated[:max_diffs]:
+              <h5>${model} <em>updated</em> in ${target_title}: ${obj}</h5>
+              <% diff = make_diff(target_data, source_data, nature="update") %>
+              <div style="padding-left: 2rem;">
+                ${diff.render_html()}
+              </div>
+          % endfor
+          % if len(updated) > max_diffs:
+              <h5>${model} - ${app.render_quantity(len(updated) - max_diffs)} more records <em>updated</em> in ${target_title} - not shown here</h5>
+          % endif
+
+          % for obj, target_data in deleted[:max_diffs]:
+              <h5>${model} <em>deleted</em> in ${target_title}: ${obj}</h5>
+              <% diff = make_diff(target_data, {}, nature="delete") %>
+              <div style="padding-left: 2rem;">
+                ${diff.render_html()}
+              </div>
+          % endfor
+          % if len(deleted) > max_diffs:
+              <h5>${model} - ${app.render_quantity(len(deleted) - max_diffs)} more records <em>deleted</em> in ${target_title} - not shown here</h5>
+          % endif
+
+        </div>
+
+    % endfor
+  </body>
+</html>

src/wuttasync/emails.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  WuttaSync -- Wutta Framework for data import/export and real-time sync
+#  Copyright © 2024-2025 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+:term:`Email Settings <email setting>` for WuttaSync
+"""
+
+import datetime
+import re
+from uuid import UUID
+
+from wuttjamaican.email import EmailSetting
+from wuttjamaican.diffs import Diff
+
+
+class ImportExportWarning(EmailSetting):
+    """
+    Base class for import/export diff warnings; sent when unexpected
+    changes occur.
+
+    This inherits from :class:`~wuttjamaican.email.EmailSetting`.
+    """
+
+    fallback_key = "import_export_warning"
+    ""  # suppress docs
+
+    import_handler_spec = None
+    import_handler_key = None
+
+    def get_description(self):  # pylint: disable=empty-docstring
+        """ """
+        handler = self.get_import_handler()
+        return f"Diff warning email for {handler.actioning} {handler.get_title()}"
+
+    def get_default_subject(self):  # pylint: disable=empty-docstring
+        """ """
+        handler = self.get_import_handler()
+        return f"Changes for {handler.get_title()}"
+
+    def get_import_handler(self):  # pylint: disable=missing-function-docstring
+
+        # prefer explicit spec, if set
+        if self.import_handler_spec:
+            return self.app.load_object(self.import_handler_spec)(self.config)
+
+        # next try spec lookup, if key set
+        if self.import_handler_key:
+            return self.app.get_import_handler(self.import_handler_key, require=True)
+
+        # or maybe try spec lookup basd on setting class name
+        class_name = self.__class__.__name__
+        if match := re.match(
+            r"^(?P<action>import|export)_to_(?P<target>\S+)_from_(?P<source>\S+)_warning$",
+            class_name,
+        ):
+            key = f"{match['action']}.to_{match['target']}.from_{match['source']}"
+            return self.app.get_import_handler(key, require=True)
+
+        raise ValueError(
+            "must set import_handler_spec (or import_handler_key) "
+            f"for email setting: {class_name}"
+        )
+
+    # nb. this is just used for sample data
+    def make_diff(self, *args, **kwargs):  # pylint: disable=missing-function-docstring
+        return Diff(self.config, *args, **kwargs)
+
+    def sample_data(self):  # pylint: disable=empty-docstring
+        """ """
+        model = self.app.model
+        handler = self.get_import_handler()
+
+        alice = model.User(username="alice")
+        bob = model.User(username="bob")
+        charlie = model.User(username="charlie")
+
+        runtime = datetime.timedelta(seconds=30)
+        return {
+            "handler": handler,
+            "title": handler.get_title(),
+            "source_title": handler.get_source_title(),
+            "target_title": handler.get_target_title(),
+            "runtime": runtime,
+            "runtime_display": "30 seconds",
+            "dry_run": True,
+            "argv": [
+                "bin/wutta",
+                "import-foo",
+                "User",
+                "--delete",
+                "--dry-run",
+                "-W",
+            ],
+            "changes": {
+                "User": (
+                    [
+                        (
+                            alice,
+                            {
+                                "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"),
+                                "username": "alice",
+                            },
+                        ),
+                    ],
+                    [
+                        (
+                            bob,
+                            {
+                                "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
+                                "username": "bob",
+                            },
+                            {
+                                "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
+                                "username": "bobbie",
+                            },
+                        ),
+                    ],
+                    [
+                        (
+                            charlie,
+                            {
+                                "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"),
+                                "username": "charlie",
+                            },
+                        ),
+                    ],
+                ),
+            },
+            "make_diff": self.make_diff,
+            "max_diffs": 15,
+        }
+
+
+class import_to_versions_from_wutta_warning(  # pylint: disable=invalid-name
+    ImportExportWarning
+):
+    """
+    Diff warning for Wutta → Versions import.
+    """
+
+
+class import_to_wutta_from_csv_warning(  # pylint: disable=invalid-name
+    ImportExportWarning
+):
+    """
+    Diff warning for CSV → Wutta import.
+    """

src/wuttasync/importing/handlers.py

@@ -26,10 +26,14 @@ Data Import / Export Handlers
 
 import logging
 import os
+import sys
 from collections import OrderedDict
 from enum import Enum
 
+import humanize
+
 from wuttjamaican.app import GenericHandler
+from wuttjamaican.diffs import Diff
 
 
 log = logging.getLogger(__name__)
@@ -44,7 +48,7 @@ class Orientation(Enum):
     EXPORT = "export"
 
 
-class ImportHandler(GenericHandler):
+class ImportHandler(GenericHandler):  # pylint: disable=too-many-public-methods
     """
     Base class for all import/export handlers.
 
@@ -121,6 +125,47 @@ class ImportHandler(GenericHandler):
     :meth:`commit_transaction()`.
     """
 
+    process_started = None
+
+    warnings = False
+    """
+    Boolean indicating the import/export should run in "warnings"
+    mode.
+
+    If set, this declares that no changes are expected for the
+    import/export job.  If any changes do occur with this flag set, a
+    diff warning email is sent within :meth:`process_changes()`.
+
+    See also :attr:`warnings_recipients`,
+    :attr:`warnings_max_diffs` and :attr:`warnings_email_key`.
+    """
+
+    warnings_email_key = None
+    """
+    Explicit :term:`email key` for sending the diff warning email,
+    *unique to this import/export type*.
+
+    Handlers do not normally set this, so the email key is determined
+    automatically within :meth:`get_warnings_email_key()`.
+
+    See also :attr:`warnings`.
+    """
+
+    warnings_recipients = None
+    """
+    Explicit recipient list for the warning email.  If not set, the
+    recipients are determined automatically via config.
+
+    See also :attr:`warnings`.
+    """
+
+    warnings_max_diffs = 15
+    """
+    Max number of record diffs (per model) to show in the warning email.
+
+    See also :attr:`warnings`.
+    """
+
     importers = None
     """
     This should be a dict of all importer/exporter classes available
@@ -164,18 +209,21 @@ class ImportHandler(GenericHandler):
     @classmethod
     def get_key(cls):
         """
-        Returns the "full key" for the handler.  This is a combination
+        Returns the :term:`import/export key` for the handler.  This
-        of :attr:`source_key` and :attr:`target_key` and
+        is a combination of :attr:`source_key` and :attr:`target_key`
-        :attr:`orientation`.
+        and :attr:`orientation`.
 
-        For instance in the case of CSV → Wutta, the full handler key
+        For instance in the case of Wutta → CSV export, the key is:
-        is ``to_wutta.from_csv.import``.
+        ``export.to_csv.from_wutta``
 
-        Note that more than one handler may return the same full key
+        Note that more than one handler may use the same key; but only
-        here; but only one will be configured as the "default" handler
+        one will be configured as the "designated" handler for that
-        for that key.  See also :meth:`get_spec()`.
+        key, a la
+        :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`.
+
+        See also :meth:`get_spec()`.
         """
-        return f"to_{cls.target_key}.from_{cls.source_key}.{cls.orientation.value}"
+        return f"{cls.orientation.value}.to_{cls.target_key}.from_{cls.source_key}"
 
     @classmethod
     def get_spec(cls):
@@ -278,11 +326,14 @@ class ImportHandler(GenericHandler):
         * :meth:`begin_transaction()`
         * :meth:`get_importer()`
         * :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter)
+        * :meth:`process_changes()`
         * :meth:`rollback_transaction()`
         * :meth:`commit_transaction()`
         """
         kwargs = self.consume_kwargs(kwargs)
+        self.process_started = self.app.localtime()
         self.begin_transaction()
+        changes = OrderedDict()
 
         success = False
         try:
@@ -293,22 +344,31 @@ class ImportHandler(GenericHandler):
                 # invoke importer
                 importer = self.get_importer(key, **kwargs)
                 created, updated, deleted = importer.process_data()
+                changed = bool(created or updated or deleted)
 
                 # log what happened
                 msg = "%s: added %d; updated %d; deleted %d %s records"
                 if self.dry_run:
                     msg += " (dry run)"
-                log.info(
+                logger = log.warning if changed and self.warnings else log.info
+                logger(
                     msg, self.get_title(), len(created), len(updated), len(deleted), key
                 )
 
+                # keep track of any changes
+                if changed:
+                    changes[key] = created, updated, deleted
+
+            # post-processing for all changes
+            if changes:
+                self.process_changes(changes)
+
+            success = True
+
         except:
             log.exception("what should happen here?")  # TODO
             raise
 
-        else:
-            success = True
-
         finally:
             if not success:
                 log.warning("something failed, so transaction was rolled back")
@@ -342,6 +402,17 @@ class ImportHandler(GenericHandler):
         if "dry_run" in kwargs:
             self.dry_run = kwargs["dry_run"]
 
+        if "warnings" in kwargs:
+            self.warnings = kwargs.pop("warnings")
+
+        if "warnings_recipients" in kwargs:
+            self.warnings_recipients = self.config.parse_list(
+                kwargs.pop("warnings_recipients")
+            )
+
+        if "warnings_max_diffs" in kwargs:
+            self.warnings_max_diffs = kwargs.pop("warnings_max_diffs")
+
         return kwargs
 
     def begin_transaction(self):
@@ -540,6 +611,113 @@ class ImportHandler(GenericHandler):
         """
         return kwargs
 
+    def process_changes(self, changes):
+        """
+        Run post-processing operations on the given changes, if
+        applicable.
+
+        This method is called by :meth:`process_data()`, if any
+        changes were made.
+
+        Default logic will send a "diff warning" email to the
+        configured recipient(s), if :attr:`warnings` mode is enabled.
+        If it is not enabled, nothing happens.
+
+        :param changes: :class:`~python:collections.OrderedDict` of
+           changes from the overall import/export job.  The structure
+           is described below.
+
+        Keys for the ``changes`` dict will be model/importer names,
+        for instance::
+
+           {
+               "Sprocket": {...},
+               "User": {...},
+           }
+
+        Value for each model key is a 3-tuple of ``(created, updated,
+        deleted)``.  Each of those elements is a list::
+
+           {
+               "Sprocket": (
+                   [...], # created
+                   [...], # updated
+                   [...], # deleted
+               ),
+           }
+
+        The list elements are always tuples, but the structure
+        varies::
+
+           {
+               "Sprocket": (
+                   [ # created, 2-tuples
+                       (obj, source_data),
+                   ],
+                   [ # updated, 3-tuples
+                       (obj, source_data, target_data),
+                   ],
+                   [ # deleted, 2-tuples
+                       (obj, target_data),
+                   ],
+               ),
+           }
+        """
+        if not self.warnings:
+            return
+
+        def make_diff(*args, **kwargs):
+            return Diff(self.config, *args, **kwargs)
+
+        runtime = self.app.localtime() - self.process_started
+        data = {
+            "handler": self,
+            "title": self.get_title(),
+            "source_title": self.get_source_title(),
+            "target_title": self.get_target_title(),
+            "dry_run": self.dry_run,
+            "argv": sys.argv,
+            "runtime": runtime,
+            "runtime_display": humanize.naturaldelta(runtime),
+            "changes": changes,
+            "make_diff": make_diff,
+            "max_diffs": self.warnings_max_diffs,
+        }
+
+        # maybe override recipients
+        kw = {}
+        if self.warnings_recipients:
+            kw["to"] = self.warnings_recipients
+            # TODO: should we in fact clear these..?
+            kw["cc"] = []
+            kw["bcc"] = []
+
+        # send the email
+        email_key = self.get_warnings_email_key()
+        self.app.send_email(email_key, data, fallback_key="import_export_warning", **kw)
+
+        log.info("%s: warning email was sent", self.get_title())
+
+    def get_warnings_email_key(self):
+        """
+        Returns the :term:`email key` to be used for sending the diff
+        warning email.
+
+        The email key should be unique to this import/export type
+        (really, the :term:`import/export key`) but not necessarily
+        unique to one handler.
+
+        If :attr:`warnings_email_key` is set, it will be used as-is.
+
+        Otherwise one is generated from :meth:`get_key()`.
+
+        :returns: Email key for diff warnings
+        """
+        if self.warnings_email_key:
+            return self.warnings_email_key
+
+        return self.get_key().replace(".", "_") + "_warning"
+
 
 class FromFileHandler(ImportHandler):
     """

src/wuttasync/importing/versions.py

@@ -297,7 +297,7 @@ class FromWuttaToVersionBase(FromWuttaMirror, ToWutta):
         if isinstance(  # pylint: disable=isinstance-second-argument-not-valid-type
             obj, self.model_class
         ):
-            data["_version"] = obj
+            data["_objref"] = obj
 
         return data
 
@@ -334,7 +334,7 @@ class FromWuttaToVersionBase(FromWuttaMirror, ToWutta):
         # when we "update" it always involves making a *new* version
         # record.  but that requires actually updating the "previous"
         # version to indicate the new version's transaction.
-        prev_version = target_data.pop("_version")
+        prev_version = target_data.pop("_objref")
         prev_version.end_transaction_id = self.continuum_txn.id
 
         return self.make_version(source_data, continuum.Operation.UPDATE)

src/wuttasync/testing.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8; -*-
+################################################################################
+#
+#  WuttaSync -- Wutta Framework for data import/export and real-time sync
+#  Copyright © 2024-2025 Lance Edgar
+#
+#  This file is part of Wutta Framework.
+#
+#  Wutta Framework is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU General Public License as published by the Free
+#  Software Foundation, either version 3 of the License, or (at your option) any
+#  later version.
+#
+#  Wutta Framework is distributed in the hope that it will be useful, but
+#  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+#  more details.
+#
+#  You should have received a copy of the GNU General Public License along with
+#  Wutta Framework.  If not, see <http://www.gnu.org/licenses/>.
+#
+################################################################################
+"""
+Testing utilities
+"""
+
+from wuttjamaican.testing import ConfigTestCase
+
+
+class ImportExportWarningTestCase(ConfigTestCase):
+    """
+    Base class for testing the import/export warning email settings.
+
+    This inherits from
+    :class:`~wuttjamaican:wuttjamaican.testing.ConfigTestCase`.
+
+    Example usage::
+
+       from wuttasync.testing import ImportExportWarningTestCase
+
+       class TestEmailSettings(ImportExportWarningTestCase):
+
+           def test_import_to_wutta_from_foo_warning(self):
+               self.do_test_preview("import_to_wutta_from_foo_warning")
+
+           def test_export_to_foo_from_wutta_warning(self):
+               self.do_test_preview("export_to_foo_from_wutta_warning")
+    """
+
+    app_title = "Wutta Poser"
+
+    def setUp(self):
+        self.setup_config()
+        self.config.setdefault("wutta.app_title", self.app_title)
+
+    def make_preview(  # pylint: disable=missing-function-docstring,unused-argument
+        self, key, mode="html"
+    ):
+        handler = self.app.get_email_handler()
+        setting = handler.get_email_setting(key)
+        context = setting.sample_data()
+        return handler.get_auto_html_body(
+            setting.key, context, fallback_key=setting.fallback_key
+        )
+
+    def do_test_preview(self, key):  # pylint: disable=missing-function-docstring
+        body = self.make_preview(key, mode="html")
+        self.assertIn("Diff warning for ", body)

tests/importing/test_base.py

@@ -717,8 +717,9 @@ class TestFromSqlalchemy(DataTestCase):
         )
         query = imp.get_source_query()
         self.assertIsInstance(query, orm.Query)
-        self.assertEqual(len(query.selectable.froms), 1)
+        froms = query.selectable.get_final_froms()
-        table = query.selectable.froms[0]
+        self.assertEqual(len(froms), 1)
+        table = froms[0]
         self.assertEqual(table.name, "upgrade")
 
     def test_get_source_objects(self):

tests/importing/test_handlers.py

@@ -2,12 +2,18 @@
 
 from collections import OrderedDict
 from unittest.mock import patch
+from uuid import UUID
 
 from wuttjamaican.testing import DataTestCase
 
 from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy
 
 
+class FromFooToBar(mod.ImportHandler):
+    source_key = "foo"
+    target_key = "bar"
+
+
 class TestImportHandler(DataTestCase):
 
     def make_handler(self, **kwargs):
@@ -30,10 +36,10 @@ class TestImportHandler(DataTestCase):
 
     def test_get_key(self):
         handler = self.make_handler()
-        self.assertEqual(handler.get_key(), "to_None.from_None.import")
+        self.assertEqual(handler.get_key(), "import.to_None.from_None")
 
         with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"):
-            self.assertEqual(handler.get_key(), "to_wutta.from_csv.import")
+            self.assertEqual(handler.get_key(), "import.to_wutta.from_csv")
 
     def test_get_spec(self):
         handler = self.make_handler()
@@ -149,15 +155,41 @@ class TestImportHandler(DataTestCase):
         kw = {}
         result = handler.consume_kwargs(kw)
         self.assertIs(result, kw)
+        self.assertEqual(result, {})
 
-        # captures dry-run flag
+        # dry_run (not consumed)
         self.assertFalse(handler.dry_run)
         kw["dry_run"] = True
         result = handler.consume_kwargs(kw)
         self.assertIs(result, kw)
+        self.assertIn("dry_run", kw)
         self.assertTrue(kw["dry_run"])
         self.assertTrue(handler.dry_run)
 
+        # warnings (consumed)
+        self.assertFalse(handler.warnings)
+        kw["warnings"] = True
+        result = handler.consume_kwargs(kw)
+        self.assertIs(result, kw)
+        self.assertNotIn("warnings", kw)
+        self.assertTrue(handler.warnings)
+
+        # warnings_recipients (consumed)
+        self.assertIsNone(handler.warnings_recipients)
+        kw["warnings_recipients"] = "bob@example.com"
+        result = handler.consume_kwargs(kw)
+        self.assertIs(result, kw)
+        self.assertNotIn("warnings_recipients", kw)
+        self.assertEqual(handler.warnings_recipients, ["bob@example.com"])
+
+        # warnings_max_diffs (consumed)
+        self.assertEqual(handler.warnings_max_diffs, 15)
+        kw["warnings_max_diffs"] = 30
+        result = handler.consume_kwargs(kw)
+        self.assertIs(result, kw)
+        self.assertNotIn("warnings_max_diffs", kw)
+        self.assertEqual(handler.warnings_max_diffs, 30)
+
     def test_define_importers(self):
         handler = self.make_handler()
         importers = handler.define_importers()
@@ -187,6 +219,94 @@ class TestImportHandler(DataTestCase):
             KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting
         )
 
+    def test_get_warnings_email_key(self):
+        handler = FromFooToBar(self.config)
+
+        # default
+        key = handler.get_warnings_email_key()
+        self.assertEqual(key, "import_to_bar_from_foo_warning")
+
+        # override
+        handler.warnings_email_key = "from_foo_to_bar"
+        key = handler.get_warnings_email_key()
+        self.assertEqual(key, "from_foo_to_bar")
+
+    def test_process_changes(self):
+        model = self.app.model
+        handler = self.make_handler()
+        email_handler = self.app.get_email_handler()
+
+        handler.process_started = self.app.localtime()
+
+        alice = model.User(username="alice")
+        bob = model.User(username="bob")
+        charlie = model.User(username="charlie")
+
+        changes = {
+            "User": (
+                [
+                    (
+                        alice,
+                        {
+                            "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"),
+                            "username": "alice",
+                        },
+                    ),
+                ],
+                [
+                    (
+                        bob,
+                        {
+                            "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
+                            "username": "bob",
+                        },
+                        {
+                            "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
+                            "username": "bobbie",
+                        },
+                    ),
+                ],
+                [
+                    (
+                        charlie,
+                        {
+                            "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"),
+                            "username": "charlie",
+                        },
+                    ),
+                ],
+            ),
+        }
+
+        # no email if not in warnings mode
+        self.assertFalse(handler.warnings)
+        with patch.object(self.app, "send_email") as send_email:
+            handler.process_changes(changes)
+            send_email.assert_not_called()
+
+        # email sent (to default recip) if in warnings mode
+        handler.warnings = True
+        self.config.setdefault("wutta.email.default.to", "admin@example.com")
+        with patch.object(email_handler, "deliver_message") as deliver_message:
+            handler.process_changes(changes)
+            deliver_message.assert_called_once()
+            args, kwargs = deliver_message.call_args
+            self.assertEqual(kwargs, {"recips": None})
+            self.assertEqual(len(args), 1)
+            msg = args[0]
+            self.assertEqual(msg.to, ["admin@example.com"])
+
+        # can override email recip
+        handler.warnings_recipients = ["bob@example.com"]
+        with patch.object(email_handler, "deliver_message") as deliver_message:
+            handler.process_changes(changes)
+            deliver_message.assert_called_once()
+            args, kwargs = deliver_message.call_args
+            self.assertEqual(kwargs, {"recips": None})
+            self.assertEqual(len(args), 1)
+            msg = args[0]
+            self.assertEqual(msg.to, ["bob@example.com"])
+
 
 class TestFromFileHandler(DataTestCase):
 

tests/importing/test_versions.py

@@ -132,8 +132,8 @@ class TestFromWuttaToVersionBase(VersionTestCase):
         # version object should be embedded in data dict
         data = imp.normalize_target_object(version)
         self.assertIsInstance(data, dict)
-        self.assertIn("_version", data)
+        self.assertIn("_objref", data)
-        self.assertIs(data["_version"], version)
+        self.assertIs(data["_objref"], version)
 
         # but normal object is not embedded
         data = imp.normalize_target_object(user)

tests/test_app.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8; -*-
+
+from wuttjamaican.testing import ConfigTestCase
+
+from wuttasync import app as mod
+from wuttasync.importing import ImportHandler
+from wuttasync.importing.csv import FromCsvToWutta
+
+
+class FromFooToBar(ImportHandler):
+    source_key = "foo"
+    target_key = "bar"
+
+
+class FromCsvToPoser(FromCsvToWutta):
+    pass
+
+
+class TestWuttaSyncAppProvider(ConfigTestCase):
+
+    def test_get_all_import_handlers(self):
+
+        # by default our custom handler is not found
+        handlers = self.app.get_all_import_handlers()
+        self.assertIn(FromCsvToWutta, handlers)
+        self.assertNotIn(FromFooToBar, handlers)
+
+        # make sure if we configure a custom handler, it is found
+        self.config.setdefault(
+            "wuttasync.importing.import.to_wutta.from_csv.handler",
+            "tests.test_app:FromFooToBar",
+        )
+        handlers = self.app.get_all_import_handlers()
+        self.assertIn(FromCsvToWutta, handlers)
+        self.assertIn(FromFooToBar, handlers)
+
+    def test_get_designated_import_handler_spec(self):
+
+        # fetch of unknown key returns none
+        spec = self.app.get_designated_import_handler_spec("test01")
+        self.assertIsNone(spec)
+
+        # unless we require it, in which case, error
+        self.assertRaises(
+            ValueError,
+            self.app.get_designated_import_handler_spec,
+            "test01",
+            require=True,
+        )
+
+        # we configure one for whatever key we like
+        self.config.setdefault(
+            "wuttasync.importing.test02.handler", "tests.test_app:FromBarToFoo"
+        )
+        spec = self.app.get_designated_import_handler_spec("test02")
+        self.assertEqual(spec, "tests.test_app:FromBarToFoo")
+
+        # we can also define a "default" designated handler
+        self.config.setdefault(
+            "wuttasync.importing.test03.default_handler",
+            "tests.test_app:FromBarToFoo",
+        )
+        spec = self.app.get_designated_import_handler_spec("test03")
+        self.assertEqual(spec, "tests.test_app:FromBarToFoo")
+
+    def test_get_designated_import_handlers(self):
+
+        # some designated handlers exist, but not our custom handler
+        handlers = self.app.get_designated_import_handlers()
+        csv_handlers = [
+            h for h in handlers if h.get_key() == "import.to_wutta.from_csv"
+        ]
+        self.assertEqual(len(csv_handlers), 1)
+        csv_handler = csv_handlers[0]
+        self.assertIsInstance(csv_handler, FromCsvToWutta)
+        self.assertFalse(isinstance(csv_handler, FromCsvToPoser))
+        self.assertFalse(
+            any([h.get_key() == "import.to_bar.from_foo" for h in handlers])
+        )
+        self.assertFalse(any([isinstance(h, FromFooToBar) for h in handlers]))
+        self.assertFalse(any([isinstance(h, FromCsvToPoser) for h in handlers]))
+        self.assertTrue(
+            any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
+        )
+
+        # but we can make custom designated
+        self.config.setdefault(
+            "wuttasync.importing.import.to_wutta.from_csv.handler",
+            "tests.test_app:FromCsvToPoser",
+        )
+        handlers = self.app.get_designated_import_handlers()
+        csv_handlers = [
+            h for h in handlers if h.get_key() == "import.to_wutta.from_csv"
+        ]
+        self.assertEqual(len(csv_handlers), 1)
+        csv_handler = csv_handlers[0]
+        self.assertIsInstance(csv_handler, FromCsvToWutta)
+        self.assertIsInstance(csv_handler, FromCsvToPoser)
+        self.assertTrue(
+            any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
+        )
+
+    def test_get_import_handler(self):
+
+        # make sure a basic fetch works
+        handler = self.app.get_import_handler("import.to_wutta.from_csv")
+        self.assertIsInstance(handler, FromCsvToWutta)
+        self.assertFalse(isinstance(handler, FromCsvToPoser))
+
+        # and make sure custom override works
+        self.config.setdefault(
+            "wuttasync.importing.import.to_wutta.from_csv.handler",
+            "tests.test_app:FromCsvToPoser",
+        )
+        handler = self.app.get_import_handler("import.to_wutta.from_csv")
+        self.assertIsInstance(handler, FromCsvToWutta)
+        self.assertIsInstance(handler, FromCsvToPoser)
+
+        # unknown importer cannot be found
+        handler = self.app.get_import_handler("bogus")
+        self.assertIsNone(handler)
+
+        # and if we require it, error will raise
+        self.assertRaises(
+            ValueError, self.app.get_import_handler, "bogus", require=True
+        )

tests/test_emails.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8; -*-
+
+from wuttjamaican.testing import ConfigTestCase
+
+from wuttasync import emails as mod
+from wuttasync.importing import ImportHandler
+from wuttasync.testing import ImportExportWarningTestCase
+
+
+class FromFooToWutta(ImportHandler):
+    pass
+
+
+class TestImportExportWarning(ConfigTestCase):
+
+    def make_setting(self, factory=None):
+        if not factory:
+            factory = mod.ImportExportWarning
+        setting = factory(self.config)
+        return setting
+
+    def test_get_description(self):
+        self.config.setdefault("wutta.app_title", "Wutta Poser")
+        setting = self.make_setting()
+        setting.import_handler_key = "import.to_wutta.from_csv"
+        self.assertEqual(
+            setting.get_description(),
+            "Diff warning email for importing CSV → Wutta Poser",
+        )
+
+    def test_get_default_subject(self):
+        self.config.setdefault("wutta.app_title", "Wutta Poser")
+        setting = self.make_setting()
+        setting.import_handler_key = "import.to_wutta.from_csv"
+        self.assertEqual(setting.get_default_subject(), "Changes for CSV → Wutta Poser")
+
+    def test_get_import_handler(self):
+
+        # nb. typical name pattern
+        class import_to_wutta_from_foo_warning(mod.ImportExportWarning):
+            pass
+
+        # nb. name does not match spec pattern
+        class import_to_wutta_from_bar_blah(mod.ImportExportWarning):
+            pass
+
+        # register our import handler
+        self.config.setdefault(
+            "wuttasync.importing.import.to_wutta.from_foo.handler",
+            "tests.test_emails:FromFooToWutta",
+        )
+
+        # error if spec/key not discoverable
+        setting = self.make_setting(import_to_wutta_from_bar_blah)
+        self.assertRaises(ValueError, setting.get_import_handler)
+
+        # can lookup by name (auto-spec)
+        setting = self.make_setting(import_to_wutta_from_foo_warning)
+        handler = setting.get_import_handler()
+        self.assertIsInstance(handler, FromFooToWutta)
+
+        # can lookup by explicit spec
+        setting = self.make_setting(import_to_wutta_from_bar_blah)
+        setting.import_handler_spec = "tests.test_emails:FromFooToWutta"
+        handler = setting.get_import_handler()
+        self.assertIsInstance(handler, FromFooToWutta)
+
+        # can lookup by explicit key
+        setting = self.make_setting(import_to_wutta_from_bar_blah)
+        setting.import_handler_key = "import.to_wutta.from_foo"
+        handler = setting.get_import_handler()
+        self.assertIsInstance(handler, FromFooToWutta)
+
+
+class TestEmailSettings(ImportExportWarningTestCase):
+
+    def test_import_to_versions_from_wutta_warning(self):
+        self.do_test_preview("import_to_versions_from_wutta_warning")
+
+    def test_import_to_wutta_from_csv_warning(self):
+        self.do_test_preview("import_to_wutta_from_csv_warning")