diff --git a/docs/api/wuttasync.app.rst b/docs/api/wuttasync.app.rst new file mode 100644 index 0000000..90f3fa7 --- /dev/null +++ b/docs/api/wuttasync.app.rst @@ -0,0 +1,6 @@ + +``wuttasync.app`` +================= + +.. automodule:: wuttasync.app + :members: diff --git a/docs/api/wuttasync.emails.rst b/docs/api/wuttasync.emails.rst new file mode 100644 index 0000000..63bf435 --- /dev/null +++ b/docs/api/wuttasync.emails.rst @@ -0,0 +1,6 @@ + +``wuttasync.emails`` +==================== + +.. automodule:: wuttasync.emails + :members: diff --git a/docs/api/wuttasync.testing.rst b/docs/api/wuttasync.testing.rst new file mode 100644 index 0000000..e6f1877 --- /dev/null +++ b/docs/api/wuttasync.testing.rst @@ -0,0 +1,6 @@ + +``wuttasync.testing`` +===================== + +.. automodule:: wuttasync.testing + :members: diff --git a/docs/glossary.rst b/docs/glossary.rst index c58e3d6..6a28b11 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -6,6 +6,26 @@ Glossary .. glossary:: :sorted: + import/export key + Unique key representing a particular type of import/export job, + i.e. the source/target and orientation (import vs. export). + + For instance "Wutta → CSV export" uses the key: + ``export.to_csv.from_wutta`` + + More than one :term:`import handler` can share a key, e.g. one + may subclass another and inherit the key. + + However only one handler is "designated" for a given key; it will + be used by default for running those jobs. + + This key is used for lookup in + :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`. + + See also + :meth:`~wuttasync.importing.handlers.ImportHandler.get_key()` + method on the import/export handler. + import handler This a type of :term:`handler` which is responsible for a particular set of data import/export task(s). diff --git a/docs/index.rst b/docs/index.rst index 6fe554a..e6fea22 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -67,13 +67,15 @@ cf. :doc:`rattail-manual:data/sync/index`. .. toctree:: :maxdepth: 1 - :caption: API + :caption: Package API api/wuttasync + api/wuttasync.app api/wuttasync.cli api/wuttasync.cli.base api/wuttasync.cli.import_csv api/wuttasync.cli.import_versions + api/wuttasync.emails api/wuttasync.importing api/wuttasync.importing.base api/wuttasync.importing.csv @@ -81,4 +83,5 @@ cf. :doc:`rattail-manual:data/sync/index`. api/wuttasync.importing.model api/wuttasync.importing.versions api/wuttasync.importing.wutta + api/wuttasync.testing api/wuttasync.util diff --git a/pyproject.toml b/pyproject.toml index cff065a..51a1a70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ + "humanize", "makefun", "SQLAlchemy-Utils", "WuttJamaican[db]>=0.16.2", @@ -37,6 +38,13 @@ docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum"] +[project.entry-points."wutta.app.providers"] +wuttasync = "wuttasync.app:WuttaSyncAppProvider" + +[project.entry-points."wuttasync.importing"] +"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions" +"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta" + [project.entry-points."wutta.typer_imports"] wuttasync = "wuttasync.cli" diff --git a/src/wuttasync/app.py b/src/wuttasync/app.py new file mode 100644 index 0000000..0fa19fd --- /dev/null +++ b/src/wuttasync/app.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +App handler supplement for WuttaSync +""" + +from collections import OrderedDict + +from wuttjamaican.app import AppProvider +from wuttjamaican.util import load_entry_points + + +class WuttaSyncAppProvider(AppProvider): + """ + The :term:`app provider` for WuttaSync. + + This adds some methods to the :term:`app handler`, which are + specific to import/export. + + It also declares some :term:`email modules ` and + :term:`email templates ` for the app. + + We have two concerns when doing lookups etc. for import/export + handlers: + + * which handlers are *available* - i.e. they exist and are + discoverable + * which handlers are *designated* - only one designated handler + per key + + All "available" handlers will have a key, but some keys may be + referenced by multiple handlers. For each key, only one handler + can be "designated" - there is a default, but config can override. + """ + + email_modules = ["wuttasync.emails"] + email_templates = ["wuttasync:email-templates"] + + def get_all_import_handlers(self): + """ + Returns *all* :term:`import/export handler ` + *classes* which are known to exist, i.e. are discoverable. + + See also :meth:`get_import_handler()` and + :meth:`get_designated_import_handlers()`. + + The discovery process is as follows: + + * load handlers from registered entry points + * check config for designated handlers + + Checking for designated handler config is not a reliable way + to discover handlers, but it's done just in case any new ones + might be found. + + Registration via entry points is the only way to ensure a + handler is discoverable. The entry point group name is always + ``wuttasync.importing`` regardless of :term:`app name`; + entries are like ``"handler_key" = "handler_spec"``. For + example: + + .. code-block:: toml + + [project.entry-points."wuttasync.importing"] + "export.to_csv.from_poser" = "poser.exporting.csv:FromPoserToCsv" + "import.to_poser.from_csv" = "poser.importing.csv:FromCsvToPoser" + + :returns: List of all import/export handler classes + """ + # first load all "registered" Handler classes + factories = load_entry_points("wuttasync.importing", ignore_errors=True) + + # organize registered classes by spec + specs = {factory.get_spec(): factory for factory in factories.values()} + + # many handlers may not be registered per se, but may be + # designated via config. so try to include those too + for factory in factories.values(): + spec = self.get_designated_import_handler_spec(factory.get_key()) + if spec and spec not in specs: + specs[spec] = self.app.load_object(spec) + + # flatten back to simple list of classes + factories = list(specs.values()) + return factories + + def get_designated_import_handler_spec(self, key, require=False): + """ + Returns the designated import/export handler :term:`spec` + string for the given type key. + + This just checks config for the designated handler, using the + ``wuttasync.importing`` prefix regardless of :term:`app name`. + For instance: + + .. code-block:: ini + + [wuttasync.importing] + export.to_csv.from_poser.handler = poser.exporting.csv:FromPoserToCsv + import.to_poser.from_csv.handler = poser.importing.csv:FromCsvToPoser + + See also :meth:`get_designated_import_handlers()` and + :meth:`get_import_handler()`. + + :param key: Unique key indicating the type of import/export + handler. + + :param require: Flag indicating whether an error should be raised if no + handler is found. + + :returns: Spec string for the designated handler. If none is + configured, then ``None`` is returned *unless* the + ``require`` param is true, in which case an error is + raised. + """ + spec = self.config.get(f"wuttasync.importing.{key}.handler") + if spec: + return spec + + spec = self.config.get(f"wuttasync.importing.{key}.default_handler") + if spec: + return spec + + if require: + raise ValueError(f"Cannot locate import handler spec for key: {key}") + return None + + def get_designated_import_handlers(self): + """ + Returns all *designated* import/export handler *instances*. + + Each import/export handler has a "key" which indicates the + "type" of import/export job it performs. For instance the CSV + → Wutta import has the key: ``import.to_wutta.from_csv`` + + More than one handler can be defined for that key; however + only one such handler will be "designated" for each key. + + This method first loads *all* available import handlers, then + organizes them by key, and tries to determine which handler + should be designated for each key. + + See also :meth:`get_all_import_handlers()` and + :meth:`get_designated_import_handler_spec()`. + + :returns: List of designated import/export handler instances + """ + grouped = OrderedDict() + for factory in self.get_all_import_handlers(): + key = factory.get_key() + grouped.setdefault(key, []).append(factory) + + def find_designated(key, group): + spec = self.get_designated_import_handler_spec(key) + if spec: + for factory in group: + if factory.get_spec() == spec: + return factory + if len(group) == 1: + return group[0] + return None + + designated = [] + for key, group in grouped.items(): + factory = find_designated(key, group) + if factory: + handler = factory(self.config) + designated.append(handler) + + return designated + + def get_import_handler(self, key, require=False, **kwargs): + """ + Returns the designated :term:`import/export handler ` instance for the given :term:`import/export key`. + + See also :meth:`get_all_import_handlers()` and + :meth:`get_designated_import_handlers()`. + + :param key: Key indicating the type of import/export handler, + e.g. ``"import.to_wutta.from_csv"`` + + :param require: Set this to true if you want an error raised + when no handler is found. + + :returns: The import/export handler instance. If no handler + is found, then ``None`` is returned, unless ``require`` + param is true, in which case error is raised. + """ + # first try to fetch the handler per designated spec + spec = self.get_designated_import_handler_spec(key, **kwargs) + if spec: + factory = self.app.load_object(spec) + return factory(self.config) + + # nothing was designated, so leverage logic which already + # sorts out which handler is "designated" for given key + designated = self.get_designated_import_handlers() + for handler in designated: + if handler.get_key() == key: + return handler + + if require: + raise ValueError(f"Cannot locate import handler for key: {key}") + return None diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index 08fa4f5..a2460d5 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -127,15 +127,16 @@ class ImportCommandHandler(GenericHandler): This is what happens when command line has ``--list-models``. """ - sys.stdout.write("ALL MODELS:\n") + sys.stdout.write("\nALL MODELS:\n") sys.stdout.write("==============================\n") for key in self.import_handler.importers: sys.stdout.write(key) sys.stdout.write("\n") sys.stdout.write("==============================\n") + sys.stdout.write(f"for {self.import_handler.get_title()}\n\n") -def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments +def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments,too-many-locals models: Annotated[ Optional[List[str]], typer.Argument( @@ -217,6 +218,27 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen help="Max number of *any* target record changes which may occur (per model)." ), ] = None, + warnings: Annotated[ + bool, + typer.Option( + "--warn", + "-W", + help="Expect no changes; warn (email the diff) if any occur.", + ), + ] = False, + warnings_recipients: Annotated[ + str, + typer.Option( + "--recip", help="Override the recipient(s) for diff warning email." + ), + ] = None, + warnings_max_diffs: Annotated[ + int, + typer.Option( + "--max-diffs", + help="Max number of record diffs to show (per model) in warning email.", + ), + ] = 15, dry_run: Annotated[ bool, typer.Option( diff --git a/src/wuttasync/email-templates/import_export_warning.html.mako b/src/wuttasync/email-templates/import_export_warning.html.mako new file mode 100644 index 0000000..9be7770 --- /dev/null +++ b/src/wuttasync/email-templates/import_export_warning.html.mako @@ -0,0 +1,88 @@ +## -*- coding: utf-8; -*- + + +

Diff warning for ${title} (${handler.actioning})

+ +

+ % if dry_run: + DRY RUN + - these changes have not yet happened + % else: + LIVE RUN + - these changes already happened + % endif +

+ +
    + % for model, (created, updated, deleted) in changes.items(): +
  • + ${model} - + ${app.render_quantity(len(created))} created; + ${app.render_quantity(len(updated))} updated; + ${app.render_quantity(len(deleted))} deleted +
    • + % endfor +
+ +

+ COMMAND: +   + ${argv} +

+ +

+ RUNTIME: +   + ${runtime} (${runtime_display}) +

+ + % for model, (created, updated, deleted) in changes.items(): + +
+

+ ${model} - + ${app.render_quantity(len(created))} created; + ${app.render_quantity(len(updated))} updated; + ${app.render_quantity(len(deleted))} deleted +

+ +
+ + % for obj, source_data in created[:max_diffs]: +
${model} created in ${target_title}: ${obj}
+ <% diff = make_diff({}, source_data, nature="create") %> +
+ ${diff.render_html()} +
+ % endfor + % if len(created) > max_diffs: +
${model} - ${app.render_quantity(len(created) - max_diffs)} more records created in ${target_title} - not shown here
+ % endif + + % for obj, source_data, target_data in updated[:max_diffs]: +
${model} updated in ${target_title}: ${obj}
+ <% diff = make_diff(target_data, source_data, nature="update") %> +
+ ${diff.render_html()} +
+ % endfor + % if len(updated) > max_diffs: +
${model} - ${app.render_quantity(len(updated) - max_diffs)} more records updated in ${target_title} - not shown here
+ % endif + + % for obj, target_data in deleted[:max_diffs]: +
${model} deleted in ${target_title}: ${obj}
+ <% diff = make_diff(target_data, {}, nature="delete") %> +
+ ${diff.render_html()} +
+ % endfor + % if len(deleted) > max_diffs: +
${model} - ${app.render_quantity(len(deleted) - max_diffs)} more records deleted in ${target_title} - not shown here
+ % endif + +
+ + % endfor + + diff --git a/src/wuttasync/emails.py b/src/wuttasync/emails.py new file mode 100644 index 0000000..b34112d --- /dev/null +++ b/src/wuttasync/emails.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +:term:`Email Settings ` for WuttaSync +""" + +import datetime +import re +from uuid import UUID + +from wuttjamaican.email import EmailSetting +from wuttjamaican.diffs import Diff + + +class ImportExportWarning(EmailSetting): + """ + Base class for import/export diff warnings; sent when unexpected + changes occur. + + This inherits from :class:`~wuttjamaican.email.EmailSetting`. + """ + + fallback_key = "import_export_warning" + "" # suppress docs + + import_handler_spec = None + import_handler_key = None + + def get_description(self): # pylint: disable=empty-docstring + """ """ + handler = self.get_import_handler() + return f"Diff warning email for {handler.actioning} {handler.get_title()}" + + def get_default_subject(self): # pylint: disable=empty-docstring + """ """ + handler = self.get_import_handler() + return f"Changes for {handler.get_title()}" + + def get_import_handler(self): # pylint: disable=missing-function-docstring + + # prefer explicit spec, if set + if self.import_handler_spec: + return self.app.load_object(self.import_handler_spec)(self.config) + + # next try spec lookup, if key set + if self.import_handler_key: + return self.app.get_import_handler(self.import_handler_key, require=True) + + # or maybe try spec lookup basd on setting class name + class_name = self.__class__.__name__ + if match := re.match( + r"^(?Pimport|export)_to_(?P\S+)_from_(?P\S+)_warning$", + class_name, + ): + key = f"{match['action']}.to_{match['target']}.from_{match['source']}" + return self.app.get_import_handler(key, require=True) + + raise ValueError( + "must set import_handler_spec (or import_handler_key) " + f"for email setting: {class_name}" + ) + + # nb. this is just used for sample data + def make_diff(self, *args, **kwargs): # pylint: disable=missing-function-docstring + return Diff(self.config, *args, **kwargs) + + def sample_data(self): # pylint: disable=empty-docstring + """ """ + model = self.app.model + handler = self.get_import_handler() + + alice = model.User(username="alice") + bob = model.User(username="bob") + charlie = model.User(username="charlie") + + runtime = datetime.timedelta(seconds=30) + return { + "handler": handler, + "title": handler.get_title(), + "source_title": handler.get_source_title(), + "target_title": handler.get_target_title(), + "runtime": runtime, + "runtime_display": "30 seconds", + "dry_run": True, + "argv": [ + "bin/wutta", + "import-foo", + "User", + "--delete", + "--dry-run", + "-W", + ], + "changes": { + "User": ( + [ + ( + alice, + { + "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"), + "username": "alice", + }, + ), + ], + [ + ( + bob, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bob", + }, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bobbie", + }, + ), + ], + [ + ( + charlie, + { + "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"), + "username": "charlie", + }, + ), + ], + ), + }, + "make_diff": self.make_diff, + "max_diffs": 15, + } + + +class import_to_versions_from_wutta_warning( # pylint: disable=invalid-name + ImportExportWarning +): + """ + Diff warning for Wutta → Versions import. + """ + + +class import_to_wutta_from_csv_warning( # pylint: disable=invalid-name + ImportExportWarning +): + """ + Diff warning for CSV → Wutta import. + """ diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index c1f7595..2a0ba71 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -26,10 +26,14 @@ Data Import / Export Handlers import logging import os +import sys from collections import OrderedDict from enum import Enum +import humanize + from wuttjamaican.app import GenericHandler +from wuttjamaican.diffs import Diff log = logging.getLogger(__name__) @@ -44,7 +48,7 @@ class Orientation(Enum): EXPORT = "export" -class ImportHandler(GenericHandler): +class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods """ Base class for all import/export handlers. @@ -121,6 +125,47 @@ class ImportHandler(GenericHandler): :meth:`commit_transaction()`. """ + process_started = None + + warnings = False + """ + Boolean indicating the import/export should run in "warnings" + mode. + + If set, this declares that no changes are expected for the + import/export job. If any changes do occur with this flag set, a + diff warning email is sent within :meth:`process_changes()`. + + See also :attr:`warnings_recipients`, + :attr:`warnings_max_diffs` and :attr:`warnings_email_key`. + """ + + warnings_email_key = None + """ + Explicit :term:`email key` for sending the diff warning email, + *unique to this import/export type*. + + Handlers do not normally set this, so the email key is determined + automatically within :meth:`get_warnings_email_key()`. + + See also :attr:`warnings`. + """ + + warnings_recipients = None + """ + Explicit recipient list for the warning email. If not set, the + recipients are determined automatically via config. + + See also :attr:`warnings`. + """ + + warnings_max_diffs = 15 + """ + Max number of record diffs (per model) to show in the warning email. + + See also :attr:`warnings`. + """ + importers = None """ This should be a dict of all importer/exporter classes available @@ -164,18 +209,21 @@ class ImportHandler(GenericHandler): @classmethod def get_key(cls): """ - Returns the "full key" for the handler. This is a combination - of :attr:`source_key` and :attr:`target_key` and - :attr:`orientation`. + Returns the :term:`import/export key` for the handler. This + is a combination of :attr:`source_key` and :attr:`target_key` + and :attr:`orientation`. - For instance in the case of CSV → Wutta, the full handler key - is ``to_wutta.from_csv.import``. + For instance in the case of Wutta → CSV export, the key is: + ``export.to_csv.from_wutta`` - Note that more than one handler may return the same full key - here; but only one will be configured as the "default" handler - for that key. See also :meth:`get_spec()`. + Note that more than one handler may use the same key; but only + one will be configured as the "designated" handler for that + key, a la + :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`. + + See also :meth:`get_spec()`. """ - return f"to_{cls.target_key}.from_{cls.source_key}.{cls.orientation.value}" + return f"{cls.orientation.value}.to_{cls.target_key}.from_{cls.source_key}" @classmethod def get_spec(cls): @@ -278,11 +326,14 @@ class ImportHandler(GenericHandler): * :meth:`begin_transaction()` * :meth:`get_importer()` * :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter) + * :meth:`process_changes()` * :meth:`rollback_transaction()` * :meth:`commit_transaction()` """ kwargs = self.consume_kwargs(kwargs) + self.process_started = self.app.localtime() self.begin_transaction() + changes = OrderedDict() success = False try: @@ -293,22 +344,31 @@ class ImportHandler(GenericHandler): # invoke importer importer = self.get_importer(key, **kwargs) created, updated, deleted = importer.process_data() + changed = bool(created or updated or deleted) # log what happened msg = "%s: added %d; updated %d; deleted %d %s records" if self.dry_run: msg += " (dry run)" - log.info( + logger = log.warning if changed and self.warnings else log.info + logger( msg, self.get_title(), len(created), len(updated), len(deleted), key ) + # keep track of any changes + if changed: + changes[key] = created, updated, deleted + + # post-processing for all changes + if changes: + self.process_changes(changes) + + success = True + except: log.exception("what should happen here?") # TODO raise - else: - success = True - finally: if not success: log.warning("something failed, so transaction was rolled back") @@ -342,6 +402,17 @@ class ImportHandler(GenericHandler): if "dry_run" in kwargs: self.dry_run = kwargs["dry_run"] + if "warnings" in kwargs: + self.warnings = kwargs.pop("warnings") + + if "warnings_recipients" in kwargs: + self.warnings_recipients = self.config.parse_list( + kwargs.pop("warnings_recipients") + ) + + if "warnings_max_diffs" in kwargs: + self.warnings_max_diffs = kwargs.pop("warnings_max_diffs") + return kwargs def begin_transaction(self): @@ -540,6 +611,113 @@ class ImportHandler(GenericHandler): """ return kwargs + def process_changes(self, changes): + """ + Run post-processing operations on the given changes, if + applicable. + + This method is called by :meth:`process_data()`, if any + changes were made. + + Default logic will send a "diff warning" email to the + configured recipient(s), if :attr:`warnings` mode is enabled. + If it is not enabled, nothing happens. + + :param changes: :class:`~python:collections.OrderedDict` of + changes from the overall import/export job. The structure + is described below. + + Keys for the ``changes`` dict will be model/importer names, + for instance:: + + { + "Sprocket": {...}, + "User": {...}, + } + + Value for each model key is a 3-tuple of ``(created, updated, + deleted)``. Each of those elements is a list:: + + { + "Sprocket": ( + [...], # created + [...], # updated + [...], # deleted + ), + } + + The list elements are always tuples, but the structure + varies:: + + { + "Sprocket": ( + [ # created, 2-tuples + (obj, source_data), + ], + [ # updated, 3-tuples + (obj, source_data, target_data), + ], + [ # deleted, 2-tuples + (obj, target_data), + ], + ), + } + """ + if not self.warnings: + return + + def make_diff(*args, **kwargs): + return Diff(self.config, *args, **kwargs) + + runtime = self.app.localtime() - self.process_started + data = { + "handler": self, + "title": self.get_title(), + "source_title": self.get_source_title(), + "target_title": self.get_target_title(), + "dry_run": self.dry_run, + "argv": sys.argv, + "runtime": runtime, + "runtime_display": humanize.naturaldelta(runtime), + "changes": changes, + "make_diff": make_diff, + "max_diffs": self.warnings_max_diffs, + } + + # maybe override recipients + kw = {} + if self.warnings_recipients: + kw["to"] = self.warnings_recipients + # TODO: should we in fact clear these..? + kw["cc"] = [] + kw["bcc"] = [] + + # send the email + email_key = self.get_warnings_email_key() + self.app.send_email(email_key, data, fallback_key="import_export_warning", **kw) + + log.info("%s: warning email was sent", self.get_title()) + + def get_warnings_email_key(self): + """ + Returns the :term:`email key` to be used for sending the diff + warning email. + + The email key should be unique to this import/export type + (really, the :term:`import/export key`) but not necessarily + unique to one handler. + + If :attr:`warnings_email_key` is set, it will be used as-is. + + Otherwise one is generated from :meth:`get_key()`. + + :returns: Email key for diff warnings + """ + if self.warnings_email_key: + return self.warnings_email_key + + return self.get_key().replace(".", "_") + "_warning" + class FromFileHandler(ImportHandler): """ diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py index b2fd062..cda77c9 100644 --- a/src/wuttasync/importing/versions.py +++ b/src/wuttasync/importing/versions.py @@ -297,7 +297,7 @@ class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type obj, self.model_class ): - data["_version"] = obj + data["_objref"] = obj return data @@ -334,7 +334,7 @@ class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): # when we "update" it always involves making a *new* version # record. but that requires actually updating the "previous" # version to indicate the new version's transaction. - prev_version = target_data.pop("_version") + prev_version = target_data.pop("_objref") prev_version.end_transaction_id = self.continuum_txn.id return self.make_version(source_data, continuum.Operation.UPDATE) diff --git a/src/wuttasync/testing.py b/src/wuttasync/testing.py new file mode 100644 index 0000000..1daad1f --- /dev/null +++ b/src/wuttasync/testing.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# WuttaSync -- Wutta Framework for data import/export and real-time sync +# Copyright © 2024-2025 Lance Edgar +# +# This file is part of Wutta Framework. +# +# Wutta Framework is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Wutta Framework is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Wutta Framework. If not, see . +# +################################################################################ +""" +Testing utilities +""" + +from wuttjamaican.testing import ConfigTestCase + + +class ImportExportWarningTestCase(ConfigTestCase): + """ + Base class for testing the import/export warning email settings. + + This inherits from + :class:`~wuttjamaican:wuttjamaican.testing.ConfigTestCase`. + + Example usage:: + + from wuttasync.testing import ImportExportWarningTestCase + + class TestEmailSettings(ImportExportWarningTestCase): + + def test_import_to_wutta_from_foo_warning(self): + self.do_test_preview("import_to_wutta_from_foo_warning") + + def test_export_to_foo_from_wutta_warning(self): + self.do_test_preview("export_to_foo_from_wutta_warning") + """ + + app_title = "Wutta Poser" + + def setUp(self): + self.setup_config() + self.config.setdefault("wutta.app_title", self.app_title) + + def make_preview( # pylint: disable=missing-function-docstring,unused-argument + self, key, mode="html" + ): + handler = self.app.get_email_handler() + setting = handler.get_email_setting(key) + context = setting.sample_data() + return handler.get_auto_html_body( + setting.key, context, fallback_key=setting.fallback_key + ) + + def do_test_preview(self, key): # pylint: disable=missing-function-docstring + body = self.make_preview(key, mode="html") + self.assertIn("Diff warning for ", body) diff --git a/tests/importing/test_base.py b/tests/importing/test_base.py index c920ed6..9f83ae3 100644 --- a/tests/importing/test_base.py +++ b/tests/importing/test_base.py @@ -717,8 +717,9 @@ class TestFromSqlalchemy(DataTestCase): ) query = imp.get_source_query() self.assertIsInstance(query, orm.Query) - self.assertEqual(len(query.selectable.froms), 1) - table = query.selectable.froms[0] + froms = query.selectable.get_final_froms() + self.assertEqual(len(froms), 1) + table = froms[0] self.assertEqual(table.name, "upgrade") def test_get_source_objects(self): diff --git a/tests/importing/test_handlers.py b/tests/importing/test_handlers.py index a6df032..c01b405 100644 --- a/tests/importing/test_handlers.py +++ b/tests/importing/test_handlers.py @@ -2,12 +2,18 @@ from collections import OrderedDict from unittest.mock import patch +from uuid import UUID from wuttjamaican.testing import DataTestCase from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy +class FromFooToBar(mod.ImportHandler): + source_key = "foo" + target_key = "bar" + + class TestImportHandler(DataTestCase): def make_handler(self, **kwargs): @@ -30,10 +36,10 @@ class TestImportHandler(DataTestCase): def test_get_key(self): handler = self.make_handler() - self.assertEqual(handler.get_key(), "to_None.from_None.import") + self.assertEqual(handler.get_key(), "import.to_None.from_None") with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"): - self.assertEqual(handler.get_key(), "to_wutta.from_csv.import") + self.assertEqual(handler.get_key(), "import.to_wutta.from_csv") def test_get_spec(self): handler = self.make_handler() @@ -149,15 +155,41 @@ class TestImportHandler(DataTestCase): kw = {} result = handler.consume_kwargs(kw) self.assertIs(result, kw) + self.assertEqual(result, {}) - # captures dry-run flag + # dry_run (not consumed) self.assertFalse(handler.dry_run) kw["dry_run"] = True result = handler.consume_kwargs(kw) self.assertIs(result, kw) + self.assertIn("dry_run", kw) self.assertTrue(kw["dry_run"]) self.assertTrue(handler.dry_run) + # warnings (consumed) + self.assertFalse(handler.warnings) + kw["warnings"] = True + result = handler.consume_kwargs(kw) + self.assertIs(result, kw) + self.assertNotIn("warnings", kw) + self.assertTrue(handler.warnings) + + # warnings_recipients (consumed) + self.assertIsNone(handler.warnings_recipients) + kw["warnings_recipients"] = "bob@example.com" + result = handler.consume_kwargs(kw) + self.assertIs(result, kw) + self.assertNotIn("warnings_recipients", kw) + self.assertEqual(handler.warnings_recipients, ["bob@example.com"]) + + # warnings_max_diffs (consumed) + self.assertEqual(handler.warnings_max_diffs, 15) + kw["warnings_max_diffs"] = 30 + result = handler.consume_kwargs(kw) + self.assertIs(result, kw) + self.assertNotIn("warnings_max_diffs", kw) + self.assertEqual(handler.warnings_max_diffs, 30) + def test_define_importers(self): handler = self.make_handler() importers = handler.define_importers() @@ -187,6 +219,94 @@ class TestImportHandler(DataTestCase): KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting ) + def test_get_warnings_email_key(self): + handler = FromFooToBar(self.config) + + # default + key = handler.get_warnings_email_key() + self.assertEqual(key, "import_to_bar_from_foo_warning") + + # override + handler.warnings_email_key = "from_foo_to_bar" + key = handler.get_warnings_email_key() + self.assertEqual(key, "from_foo_to_bar") + + def test_process_changes(self): + model = self.app.model + handler = self.make_handler() + email_handler = self.app.get_email_handler() + + handler.process_started = self.app.localtime() + + alice = model.User(username="alice") + bob = model.User(username="bob") + charlie = model.User(username="charlie") + + changes = { + "User": ( + [ + ( + alice, + { + "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"), + "username": "alice", + }, + ), + ], + [ + ( + bob, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bob", + }, + { + "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), + "username": "bobbie", + }, + ), + ], + [ + ( + charlie, + { + "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"), + "username": "charlie", + }, + ), + ], + ), + } + + # no email if not in warnings mode + self.assertFalse(handler.warnings) + with patch.object(self.app, "send_email") as send_email: + handler.process_changes(changes) + send_email.assert_not_called() + + # email sent (to default recip) if in warnings mode + handler.warnings = True + self.config.setdefault("wutta.email.default.to", "admin@example.com") + with patch.object(email_handler, "deliver_message") as deliver_message: + handler.process_changes(changes) + deliver_message.assert_called_once() + args, kwargs = deliver_message.call_args + self.assertEqual(kwargs, {"recips": None}) + self.assertEqual(len(args), 1) + msg = args[0] + self.assertEqual(msg.to, ["admin@example.com"]) + + # can override email recip + handler.warnings_recipients = ["bob@example.com"] + with patch.object(email_handler, "deliver_message") as deliver_message: + handler.process_changes(changes) + deliver_message.assert_called_once() + args, kwargs = deliver_message.call_args + self.assertEqual(kwargs, {"recips": None}) + self.assertEqual(len(args), 1) + msg = args[0] + self.assertEqual(msg.to, ["bob@example.com"]) + class TestFromFileHandler(DataTestCase): diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py index 1988706..2cd4ec0 100644 --- a/tests/importing/test_versions.py +++ b/tests/importing/test_versions.py @@ -132,8 +132,8 @@ class TestFromWuttaToVersionBase(VersionTestCase): # version object should be embedded in data dict data = imp.normalize_target_object(version) self.assertIsInstance(data, dict) - self.assertIn("_version", data) - self.assertIs(data["_version"], version) + self.assertIn("_objref", data) + self.assertIs(data["_objref"], version) # but normal object is not embedded data = imp.normalize_target_object(user) diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..ea2b5e6 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8; -*- + +from wuttjamaican.testing import ConfigTestCase + +from wuttasync import app as mod +from wuttasync.importing import ImportHandler +from wuttasync.importing.csv import FromCsvToWutta + + +class FromFooToBar(ImportHandler): + source_key = "foo" + target_key = "bar" + + +class FromCsvToPoser(FromCsvToWutta): + pass + + +class TestWuttaSyncAppProvider(ConfigTestCase): + + def test_get_all_import_handlers(self): + + # by default our custom handler is not found + handlers = self.app.get_all_import_handlers() + self.assertIn(FromCsvToWutta, handlers) + self.assertNotIn(FromFooToBar, handlers) + + # make sure if we configure a custom handler, it is found + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_csv.handler", + "tests.test_app:FromFooToBar", + ) + handlers = self.app.get_all_import_handlers() + self.assertIn(FromCsvToWutta, handlers) + self.assertIn(FromFooToBar, handlers) + + def test_get_designated_import_handler_spec(self): + + # fetch of unknown key returns none + spec = self.app.get_designated_import_handler_spec("test01") + self.assertIsNone(spec) + + # unless we require it, in which case, error + self.assertRaises( + ValueError, + self.app.get_designated_import_handler_spec, + "test01", + require=True, + ) + + # we configure one for whatever key we like + self.config.setdefault( + "wuttasync.importing.test02.handler", "tests.test_app:FromBarToFoo" + ) + spec = self.app.get_designated_import_handler_spec("test02") + self.assertEqual(spec, "tests.test_app:FromBarToFoo") + + # we can also define a "default" designated handler + self.config.setdefault( + "wuttasync.importing.test03.default_handler", + "tests.test_app:FromBarToFoo", + ) + spec = self.app.get_designated_import_handler_spec("test03") + self.assertEqual(spec, "tests.test_app:FromBarToFoo") + + def test_get_designated_import_handlers(self): + + # some designated handlers exist, but not our custom handler + handlers = self.app.get_designated_import_handlers() + csv_handlers = [ + h for h in handlers if h.get_key() == "import.to_wutta.from_csv" + ] + self.assertEqual(len(csv_handlers), 1) + csv_handler = csv_handlers[0] + self.assertIsInstance(csv_handler, FromCsvToWutta) + self.assertFalse(isinstance(csv_handler, FromCsvToPoser)) + self.assertFalse( + any([h.get_key() == "import.to_bar.from_foo" for h in handlers]) + ) + self.assertFalse(any([isinstance(h, FromFooToBar) for h in handlers])) + self.assertFalse(any([isinstance(h, FromCsvToPoser) for h in handlers])) + self.assertTrue( + any([h.get_key() == "import.to_versions.from_wutta" for h in handlers]) + ) + + # but we can make custom designated + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_csv.handler", + "tests.test_app:FromCsvToPoser", + ) + handlers = self.app.get_designated_import_handlers() + csv_handlers = [ + h for h in handlers if h.get_key() == "import.to_wutta.from_csv" + ] + self.assertEqual(len(csv_handlers), 1) + csv_handler = csv_handlers[0] + self.assertIsInstance(csv_handler, FromCsvToWutta) + self.assertIsInstance(csv_handler, FromCsvToPoser) + self.assertTrue( + any([h.get_key() == "import.to_versions.from_wutta" for h in handlers]) + ) + + def test_get_import_handler(self): + + # make sure a basic fetch works + handler = self.app.get_import_handler("import.to_wutta.from_csv") + self.assertIsInstance(handler, FromCsvToWutta) + self.assertFalse(isinstance(handler, FromCsvToPoser)) + + # and make sure custom override works + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_csv.handler", + "tests.test_app:FromCsvToPoser", + ) + handler = self.app.get_import_handler("import.to_wutta.from_csv") + self.assertIsInstance(handler, FromCsvToWutta) + self.assertIsInstance(handler, FromCsvToPoser) + + # unknown importer cannot be found + handler = self.app.get_import_handler("bogus") + self.assertIsNone(handler) + + # and if we require it, error will raise + self.assertRaises( + ValueError, self.app.get_import_handler, "bogus", require=True + ) diff --git a/tests/test_emails.py b/tests/test_emails.py new file mode 100644 index 0000000..9494753 --- /dev/null +++ b/tests/test_emails.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8; -*- + +from wuttjamaican.testing import ConfigTestCase + +from wuttasync import emails as mod +from wuttasync.importing import ImportHandler +from wuttasync.testing import ImportExportWarningTestCase + + +class FromFooToWutta(ImportHandler): + pass + + +class TestImportExportWarning(ConfigTestCase): + + def make_setting(self, factory=None): + if not factory: + factory = mod.ImportExportWarning + setting = factory(self.config) + return setting + + def test_get_description(self): + self.config.setdefault("wutta.app_title", "Wutta Poser") + setting = self.make_setting() + setting.import_handler_key = "import.to_wutta.from_csv" + self.assertEqual( + setting.get_description(), + "Diff warning email for importing CSV → Wutta Poser", + ) + + def test_get_default_subject(self): + self.config.setdefault("wutta.app_title", "Wutta Poser") + setting = self.make_setting() + setting.import_handler_key = "import.to_wutta.from_csv" + self.assertEqual(setting.get_default_subject(), "Changes for CSV → Wutta Poser") + + def test_get_import_handler(self): + + # nb. typical name pattern + class import_to_wutta_from_foo_warning(mod.ImportExportWarning): + pass + + # nb. name does not match spec pattern + class import_to_wutta_from_bar_blah(mod.ImportExportWarning): + pass + + # register our import handler + self.config.setdefault( + "wuttasync.importing.import.to_wutta.from_foo.handler", + "tests.test_emails:FromFooToWutta", + ) + + # error if spec/key not discoverable + setting = self.make_setting(import_to_wutta_from_bar_blah) + self.assertRaises(ValueError, setting.get_import_handler) + + # can lookup by name (auto-spec) + setting = self.make_setting(import_to_wutta_from_foo_warning) + handler = setting.get_import_handler() + self.assertIsInstance(handler, FromFooToWutta) + + # can lookup by explicit spec + setting = self.make_setting(import_to_wutta_from_bar_blah) + setting.import_handler_spec = "tests.test_emails:FromFooToWutta" + handler = setting.get_import_handler() + self.assertIsInstance(handler, FromFooToWutta) + + # can lookup by explicit key + setting = self.make_setting(import_to_wutta_from_bar_blah) + setting.import_handler_key = "import.to_wutta.from_foo" + handler = setting.get_import_handler() + self.assertIsInstance(handler, FromFooToWutta) + + +class TestEmailSettings(ImportExportWarningTestCase): + + def test_import_to_versions_from_wutta_warning(self): + self.do_test_preview("import_to_versions_from_wutta_warning") + + def test_import_to_wutta_from_csv_warning(self): + self.do_test_preview("import_to_wutta_from_csv_warning")