diff --git a/CHANGELOG.md b/CHANGELOG.md index ef2f351..4b1fd1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,25 +5,6 @@ All notable changes to WuttaSync will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). -## v0.3.0 (2025-12-20) - -### Feat - -- add `warnings` mode for import/export handlers, commands -- add the `import-versions` command, handler logic - -### Fix - -- run all models when none specified, for import/export commands -- allow passing just `key` to ImportCommandHandler -- add `--comment` param for `import-versions` command -- add basic data type coercion for CSV -> SQLAlchemy import -- refactor some more for tests + pylint -- refactor per pylint; add to tox -- format all code with black -- tweak logging when deleting object -- add logging when deleting target object - ## v0.2.1 (2025-06-29) ### Fix diff --git a/docs/api/wuttasync.app.rst b/docs/api/wuttasync.app.rst deleted file mode 100644 index 90f3fa7..0000000 --- a/docs/api/wuttasync.app.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttasync.app`` -================= - -.. automodule:: wuttasync.app - :members: diff --git a/docs/api/wuttasync.cli.import_versions.rst b/docs/api/wuttasync.cli.import_versions.rst deleted file mode 100644 index aeb8227..0000000 --- a/docs/api/wuttasync.cli.import_versions.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttasync.cli.import_versions`` -================================= - -.. automodule:: wuttasync.cli.import_versions - :members: diff --git a/docs/api/wuttasync.emails.rst b/docs/api/wuttasync.emails.rst deleted file mode 100644 index 63bf435..0000000 --- a/docs/api/wuttasync.emails.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttasync.emails`` -==================== - -.. automodule:: wuttasync.emails - :members: diff --git a/docs/api/wuttasync.importing.versions.rst b/docs/api/wuttasync.importing.versions.rst deleted file mode 100644 index aa970a1..0000000 --- a/docs/api/wuttasync.importing.versions.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttasync.importing.versions`` -================================ - -.. automodule:: wuttasync.importing.versions - :members: diff --git a/docs/api/wuttasync.testing.rst b/docs/api/wuttasync.testing.rst deleted file mode 100644 index e6f1877..0000000 --- a/docs/api/wuttasync.testing.rst +++ /dev/null @@ -1,6 +0,0 @@ - -``wuttasync.testing`` -===================== - -.. automodule:: wuttasync.testing - :members: diff --git a/docs/conf.py b/docs/conf.py index 7826856..2b47550 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,13 +31,6 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None), - "sqlalchemy": ("http://docs.sqlalchemy.org/en/latest/", None), - "sqlalchemy-continuum": ( - "https://sqlalchemy-continuum.readthedocs.io/en/latest/", - None, - ), - "sqlalchemy-utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None), - "wutta-continuum": ("https://docs.wuttaproject.org/wutta-continuum/", None), "wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None), } diff --git a/docs/glossary.rst b/docs/glossary.rst index 6a28b11..c58e3d6 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -6,26 +6,6 @@ Glossary .. glossary:: :sorted: - import/export key - Unique key representing a particular type of import/export job, - i.e. the source/target and orientation (import vs. export). - - For instance "Wutta → CSV export" uses the key: - ``export.to_csv.from_wutta`` - - More than one :term:`import handler` can share a key, e.g. one - may subclass another and inherit the key. - - However only one handler is "designated" for a given key; it will - be used by default for running those jobs. - - This key is used for lookup in - :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`. - - See also - :meth:`~wuttasync.importing.handlers.ImportHandler.get_key()` - method on the import/export handler. - import handler This a type of :term:`handler` which is responsible for a particular set of data import/export task(s). diff --git a/docs/index.rst b/docs/index.rst index e6fea22..9eb2d93 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -67,21 +67,16 @@ cf. :doc:`rattail-manual:data/sync/index`. .. toctree:: :maxdepth: 1 - :caption: Package API + :caption: API api/wuttasync - api/wuttasync.app api/wuttasync.cli api/wuttasync.cli.base api/wuttasync.cli.import_csv - api/wuttasync.cli.import_versions - api/wuttasync.emails api/wuttasync.importing api/wuttasync.importing.base api/wuttasync.importing.csv api/wuttasync.importing.handlers api/wuttasync.importing.model - api/wuttasync.importing.versions api/wuttasync.importing.wutta - api/wuttasync.testing api/wuttasync.util diff --git a/docs/narr/cli/builtin.rst b/docs/narr/cli/builtin.rst index ac6fb14..0630c94 100644 --- a/docs/narr/cli/builtin.rst +++ b/docs/narr/cli/builtin.rst @@ -25,24 +25,3 @@ types may not behave as expected etc. Defined in: :mod:`wuttasync.cli.import_csv` .. program-output:: wutta import-csv --help - - -.. _wutta-import-versions: - -``wutta import-versions`` -------------------------- - -Import latest data to version tables, for the Wutta :term:`app -database`. - -The purpose of this is to ensure version tables accurately reflect -the current "live" data set, for given table(s). It is only -relevant/usable if versioning is configured and enabled. For more -on that see :doc:`wutta-continuum:index`. - -This command can check/update version tables for any versioned class -in the :term:`app model`. - -Defined in: :mod:`wuttasync.cli.import_versions` - -.. program-output:: wutta import-versions --help diff --git a/pyproject.toml b/pyproject.toml index 74ab85a..a48b949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" [project] name = "WuttaSync" -version = "0.3.0" +version = "0.2.1" description = "Wutta Framework for data import/export and real-time sync" readme = "README.md" authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}] @@ -26,25 +26,17 @@ classifiers = [ ] requires-python = ">= 3.8" dependencies = [ - "humanize", "makefun", "SQLAlchemy-Utils", - "WuttJamaican[db]>=0.27.0", + "WuttJamaican[db]>=0.16.2", ] [project.optional-dependencies] docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] -tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"] +tests = ["pylint", "pytest", "pytest-cov", "tox"] -[project.entry-points."wutta.app.providers"] -wuttasync = "wuttasync.app:WuttaSyncAppProvider" - -[project.entry-points."wuttasync.importing"] -"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions" -"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta" - [project.entry-points."wutta.typer_imports"] wuttasync = "wuttasync.cli" diff --git a/src/wuttasync/app.py b/src/wuttasync/app.py deleted file mode 100644 index 0fa19fd..0000000 --- a/src/wuttasync/app.py +++ /dev/null @@ -1,225 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -App handler supplement for WuttaSync -""" - -from collections import OrderedDict - -from wuttjamaican.app import AppProvider -from wuttjamaican.util import load_entry_points - - -class WuttaSyncAppProvider(AppProvider): - """ - The :term:`app provider` for WuttaSync. - - This adds some methods to the :term:`app handler`, which are - specific to import/export. - - It also declares some :term:`email modules ` and - :term:`email templates ` for the app. - - We have two concerns when doing lookups etc. for import/export - handlers: - - * which handlers are *available* - i.e. they exist and are - discoverable - * which handlers are *designated* - only one designated handler - per key - - All "available" handlers will have a key, but some keys may be - referenced by multiple handlers. For each key, only one handler - can be "designated" - there is a default, but config can override. - """ - - email_modules = ["wuttasync.emails"] - email_templates = ["wuttasync:email-templates"] - - def get_all_import_handlers(self): - """ - Returns *all* :term:`import/export handler ` - *classes* which are known to exist, i.e. are discoverable. - - See also :meth:`get_import_handler()` and - :meth:`get_designated_import_handlers()`. - - The discovery process is as follows: - - * load handlers from registered entry points - * check config for designated handlers - - Checking for designated handler config is not a reliable way - to discover handlers, but it's done just in case any new ones - might be found. - - Registration via entry points is the only way to ensure a - handler is discoverable. The entry point group name is always - ``wuttasync.importing`` regardless of :term:`app name`; - entries are like ``"handler_key" = "handler_spec"``. For - example: - - .. code-block:: toml - - [project.entry-points."wuttasync.importing"] - "export.to_csv.from_poser" = "poser.exporting.csv:FromPoserToCsv" - "import.to_poser.from_csv" = "poser.importing.csv:FromCsvToPoser" - - :returns: List of all import/export handler classes - """ - # first load all "registered" Handler classes - factories = load_entry_points("wuttasync.importing", ignore_errors=True) - - # organize registered classes by spec - specs = {factory.get_spec(): factory for factory in factories.values()} - - # many handlers may not be registered per se, but may be - # designated via config. so try to include those too - for factory in factories.values(): - spec = self.get_designated_import_handler_spec(factory.get_key()) - if spec and spec not in specs: - specs[spec] = self.app.load_object(spec) - - # flatten back to simple list of classes - factories = list(specs.values()) - return factories - - def get_designated_import_handler_spec(self, key, require=False): - """ - Returns the designated import/export handler :term:`spec` - string for the given type key. - - This just checks config for the designated handler, using the - ``wuttasync.importing`` prefix regardless of :term:`app name`. - For instance: - - .. code-block:: ini - - [wuttasync.importing] - export.to_csv.from_poser.handler = poser.exporting.csv:FromPoserToCsv - import.to_poser.from_csv.handler = poser.importing.csv:FromCsvToPoser - - See also :meth:`get_designated_import_handlers()` and - :meth:`get_import_handler()`. - - :param key: Unique key indicating the type of import/export - handler. - - :param require: Flag indicating whether an error should be raised if no - handler is found. - - :returns: Spec string for the designated handler. If none is - configured, then ``None`` is returned *unless* the - ``require`` param is true, in which case an error is - raised. - """ - spec = self.config.get(f"wuttasync.importing.{key}.handler") - if spec: - return spec - - spec = self.config.get(f"wuttasync.importing.{key}.default_handler") - if spec: - return spec - - if require: - raise ValueError(f"Cannot locate import handler spec for key: {key}") - return None - - def get_designated_import_handlers(self): - """ - Returns all *designated* import/export handler *instances*. - - Each import/export handler has a "key" which indicates the - "type" of import/export job it performs. For instance the CSV - → Wutta import has the key: ``import.to_wutta.from_csv`` - - More than one handler can be defined for that key; however - only one such handler will be "designated" for each key. - - This method first loads *all* available import handlers, then - organizes them by key, and tries to determine which handler - should be designated for each key. - - See also :meth:`get_all_import_handlers()` and - :meth:`get_designated_import_handler_spec()`. - - :returns: List of designated import/export handler instances - """ - grouped = OrderedDict() - for factory in self.get_all_import_handlers(): - key = factory.get_key() - grouped.setdefault(key, []).append(factory) - - def find_designated(key, group): - spec = self.get_designated_import_handler_spec(key) - if spec: - for factory in group: - if factory.get_spec() == spec: - return factory - if len(group) == 1: - return group[0] - return None - - designated = [] - for key, group in grouped.items(): - factory = find_designated(key, group) - if factory: - handler = factory(self.config) - designated.append(handler) - - return designated - - def get_import_handler(self, key, require=False, **kwargs): - """ - Returns the designated :term:`import/export handler ` instance for the given :term:`import/export key`. - - See also :meth:`get_all_import_handlers()` and - :meth:`get_designated_import_handlers()`. - - :param key: Key indicating the type of import/export handler, - e.g. ``"import.to_wutta.from_csv"`` - - :param require: Set this to true if you want an error raised - when no handler is found. - - :returns: The import/export handler instance. If no handler - is found, then ``None`` is returned, unless ``require`` - param is true, in which case error is raised. - """ - # first try to fetch the handler per designated spec - spec = self.get_designated_import_handler_spec(key, **kwargs) - if spec: - factory = self.app.load_object(spec) - return factory(self.config) - - # nothing was designated, so leverage logic which already - # sorts out which handler is "designated" for given key - designated = self.get_designated_import_handlers() - for handler in designated: - if handler.get_key() == key: - return handler - - if require: - raise ValueError(f"Cannot locate import handler for key: {key}") - return None diff --git a/src/wuttasync/cli/__init__.py b/src/wuttasync/cli/__init__.py index 0d88ed4..c77a4e2 100644 --- a/src/wuttasync/cli/__init__.py +++ b/src/wuttasync/cli/__init__.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar +# Copyright © 2024 Lance Edgar # # This file is part of Wutta Framework. # @@ -34,4 +34,3 @@ from .base import import_command, file_import_command, ImportCommandHandler # nb. must bring in all modules for discovery to work from . import import_csv -from . import import_versions diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index 56d6421..08fa4f5 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -50,35 +50,19 @@ class ImportCommandHandler(GenericHandler): create this handler and call its :meth:`run()` method. This handler does not know how to import/export data, but it knows - how to make its :attr:`import_handler` do it. Likewise, the - import handler is not "CLI-aware" - so this provides the glue. + how to make its :attr:`import_handler` do it. :param import_handler: During construction, caller can specify the :attr:`import_handler` as any of: * import handler instance * import handler factory (e.g. class) - * import handler :term:`spec` + * import handler spec (cf. :func:`~wuttjamaican:wuttjamaican.util.load_object()`) - :param key: Optional :term:`import/export key` to use for handler - lookup. Only used if ``import_handler`` param is not set. + For example:: - Typical usage for custom commands will be to provide the spec:: - - handler = ImportCommandHandler( - config, "poser.importing.foo:FromFooToPoser" - ) - - Library authors may prefer to use the import/export key; this lets - the command work with any designated handler:: - - handler = ImportCommandHandler( - config, key="import.to_poser.from_foo" - ) - - See also - :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()` - which does the lookup by key. + handler = ImportCommandHandler( + config, import_handler='wuttasync.importing.csv:FromCsvToWutta') """ import_handler = None @@ -87,7 +71,7 @@ class ImportCommandHandler(GenericHandler): invoked when command runs. See also :meth:`run()`. """ - def __init__(self, config, import_handler=None, key=None): + def __init__(self, config, import_handler=None): super().__init__(config) if import_handler: @@ -99,9 +83,6 @@ class ImportCommandHandler(GenericHandler): factory = self.app.load_object(import_handler) self.import_handler = factory(self.config) - elif key: - self.import_handler = self.app.get_import_handler(key, require=True) - def run(self, params, progress=None): # pylint: disable=unused-argument """ Run the import/export job(s) based on command line params. @@ -125,15 +106,15 @@ class ImportCommandHandler(GenericHandler): return # otherwise process some data - log.debug("using handler: %s", self.import_handler.get_spec()) kw = dict(params) models = kw.pop("models") - if not models: - models = list(self.import_handler.importers) + log.debug("using handler: %s", self.import_handler.get_spec()) + # TODO: need to use all/default models if none specified + # (and should know models by now for logging purposes) log.debug( - "%s %s for models: %s", - self.import_handler.actioning, - self.import_handler.get_title(), + "running %s %s for: %s", + self.import_handler, + self.import_handler.orientation.value, ", ".join(models), ) log.debug("params are: %s", kw) @@ -146,16 +127,15 @@ class ImportCommandHandler(GenericHandler): This is what happens when command line has ``--list-models``. """ - sys.stdout.write("\nALL MODELS:\n") + sys.stdout.write("ALL MODELS:\n") sys.stdout.write("==============================\n") for key in self.import_handler.importers: sys.stdout.write(key) sys.stdout.write("\n") sys.stdout.write("==============================\n") - sys.stdout.write(f"for {self.import_handler.get_title()}\n\n") -def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments,too-many-locals +def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments models: Annotated[ Optional[List[str]], typer.Argument( @@ -237,27 +217,6 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen help="Max number of *any* target record changes which may occur (per model)." ), ] = None, - warnings: Annotated[ - bool, - typer.Option( - "--warn", - "-W", - help="Expect no changes; warn (email the diff) if any occur.", - ), - ] = False, - warnings_recipients: Annotated[ - str, - typer.Option( - "--recip", help="Override the recipient(s) for diff warning email." - ), - ] = None, - warnings_max_diffs: Annotated[ - int, - typer.Option( - "--max-diffs", - help="Max number of record diffs to show (per model) in warning email.", - ), - ] = 15, dry_run: Annotated[ bool, typer.Option( diff --git a/src/wuttasync/cli/import_csv.py b/src/wuttasync/cli/import_csv.py index 4c5694a..d3c8047 100644 --- a/src/wuttasync/cli/import_csv.py +++ b/src/wuttasync/cli/import_csv.py @@ -38,5 +38,7 @@ def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument Import data from CSV file(s) to Wutta DB """ config = ctx.parent.wutta_config - handler = ImportCommandHandler(config, key="import.to_wutta.from_csv") + handler = ImportCommandHandler( + config, import_handler="wuttasync.importing.csv:FromCsvToWutta" + ) handler.run(ctx.params) diff --git a/src/wuttasync/cli/import_versions.py b/src/wuttasync/cli/import_versions.py deleted file mode 100644 index aa82088..0000000 --- a/src/wuttasync/cli/import_versions.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -See also: :ref:`wutta-import-versions` -""" - -import sys - -import rich -import typer -from typing_extensions import Annotated - -from wuttjamaican.cli import wutta_typer - -from .base import import_command, ImportCommandHandler - - -@wutta_typer.command() -@import_command -def import_versions( # pylint: disable=unused-argument - ctx: typer.Context, - comment: Annotated[ - str, - typer.Option("--comment", "-m", help="Comment to set on the transaction."), - ] = "import catch-up versions", - **kwargs, -): - """ - Import latest data to version tables, for Wutta DB - """ - config = ctx.parent.wutta_config - app = config.get_app() - - # warn/exit if libs are not installed - try: - import wutta_continuum # pylint: disable=import-outside-toplevel,unused-import - except ImportError: # pragma: no cover - rich.print( - "\n\t[bold yellow]Wutta-Continum is not installed![/bold yellow]\n" - "\n\tIf you want it, run: pip install Wutta-Continuum\n" - ) - sys.exit(1) - - # warn/exit if feature disabled - if not app.continuum_is_enabled(): # pragma: no cover - rich.print( - "\n\t[bold yellow]Wutta-Continum is not enabled![/bold yellow]\n" - "\n\tIf you want it, see: https://docs.wuttaproject.org/wutta-continuum/\n" - ) - sys.exit(1) - - handler = ImportCommandHandler(config, key="import.to_versions.from_wutta") - handler.run(ctx.params) diff --git a/src/wuttasync/email-templates/import_export_warning.html.mako b/src/wuttasync/email-templates/import_export_warning.html.mako deleted file mode 100644 index 9be7770..0000000 --- a/src/wuttasync/email-templates/import_export_warning.html.mako +++ /dev/null @@ -1,88 +0,0 @@ -## -*- coding: utf-8; -*- - - -

Diff warning for ${title} (${handler.actioning})

- -

- % if dry_run: - DRY RUN - - these changes have not yet happened - % else: - LIVE RUN - - these changes already happened - % endif -

- -
    - % for model, (created, updated, deleted) in changes.items(): -
  • - ${model} - - ${app.render_quantity(len(created))} created; - ${app.render_quantity(len(updated))} updated; - ${app.render_quantity(len(deleted))} deleted -
    • - % endfor -
- -

- COMMAND: -   - ${argv} -

- -

- RUNTIME: -   - ${runtime} (${runtime_display}) -

- - % for model, (created, updated, deleted) in changes.items(): - -
-

- ${model} - - ${app.render_quantity(len(created))} created; - ${app.render_quantity(len(updated))} updated; - ${app.render_quantity(len(deleted))} deleted -

- -
- - % for obj, source_data in created[:max_diffs]: -
${model} created in ${target_title}: ${obj}
- <% diff = make_diff({}, source_data, nature="create") %> -
- ${diff.render_html()} -
- % endfor - % if len(created) > max_diffs: -
${model} - ${app.render_quantity(len(created) - max_diffs)} more records created in ${target_title} - not shown here
- % endif - - % for obj, source_data, target_data in updated[:max_diffs]: -
${model} updated in ${target_title}: ${obj}
- <% diff = make_diff(target_data, source_data, nature="update") %> -
- ${diff.render_html()} -
- % endfor - % if len(updated) > max_diffs: -
${model} - ${app.render_quantity(len(updated) - max_diffs)} more records updated in ${target_title} - not shown here
- % endif - - % for obj, target_data in deleted[:max_diffs]: -
${model} deleted in ${target_title}: ${obj}
- <% diff = make_diff(target_data, {}, nature="delete") %> -
- ${diff.render_html()} -
- % endfor - % if len(deleted) > max_diffs: -
${model} - ${app.render_quantity(len(deleted) - max_diffs)} more records deleted in ${target_title} - not shown here
- % endif - -
- - % endfor - - diff --git a/src/wuttasync/emails.py b/src/wuttasync/emails.py deleted file mode 100644 index b34112d..0000000 --- a/src/wuttasync/emails.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -:term:`Email Settings ` for WuttaSync -""" - -import datetime -import re -from uuid import UUID - -from wuttjamaican.email import EmailSetting -from wuttjamaican.diffs import Diff - - -class ImportExportWarning(EmailSetting): - """ - Base class for import/export diff warnings; sent when unexpected - changes occur. - - This inherits from :class:`~wuttjamaican.email.EmailSetting`. - """ - - fallback_key = "import_export_warning" - "" # suppress docs - - import_handler_spec = None - import_handler_key = None - - def get_description(self): # pylint: disable=empty-docstring - """ """ - handler = self.get_import_handler() - return f"Diff warning email for {handler.actioning} {handler.get_title()}" - - def get_default_subject(self): # pylint: disable=empty-docstring - """ """ - handler = self.get_import_handler() - return f"Changes for {handler.get_title()}" - - def get_import_handler(self): # pylint: disable=missing-function-docstring - - # prefer explicit spec, if set - if self.import_handler_spec: - return self.app.load_object(self.import_handler_spec)(self.config) - - # next try spec lookup, if key set - if self.import_handler_key: - return self.app.get_import_handler(self.import_handler_key, require=True) - - # or maybe try spec lookup basd on setting class name - class_name = self.__class__.__name__ - if match := re.match( - r"^(?Pimport|export)_to_(?P\S+)_from_(?P\S+)_warning$", - class_name, - ): - key = f"{match['action']}.to_{match['target']}.from_{match['source']}" - return self.app.get_import_handler(key, require=True) - - raise ValueError( - "must set import_handler_spec (or import_handler_key) " - f"for email setting: {class_name}" - ) - - # nb. this is just used for sample data - def make_diff(self, *args, **kwargs): # pylint: disable=missing-function-docstring - return Diff(self.config, *args, **kwargs) - - def sample_data(self): # pylint: disable=empty-docstring - """ """ - model = self.app.model - handler = self.get_import_handler() - - alice = model.User(username="alice") - bob = model.User(username="bob") - charlie = model.User(username="charlie") - - runtime = datetime.timedelta(seconds=30) - return { - "handler": handler, - "title": handler.get_title(), - "source_title": handler.get_source_title(), - "target_title": handler.get_target_title(), - "runtime": runtime, - "runtime_display": "30 seconds", - "dry_run": True, - "argv": [ - "bin/wutta", - "import-foo", - "User", - "--delete", - "--dry-run", - "-W", - ], - "changes": { - "User": ( - [ - ( - alice, - { - "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"), - "username": "alice", - }, - ), - ], - [ - ( - bob, - { - "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), - "username": "bob", - }, - { - "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), - "username": "bobbie", - }, - ), - ], - [ - ( - charlie, - { - "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"), - "username": "charlie", - }, - ), - ], - ), - }, - "make_diff": self.make_diff, - "max_diffs": 15, - } - - -class import_to_versions_from_wutta_warning( # pylint: disable=invalid-name - ImportExportWarning -): - """ - Diff warning for Wutta → Versions import. - """ - - -class import_to_wutta_from_csv_warning( # pylint: disable=invalid-name - ImportExportWarning -): - """ - Diff warning for CSV → Wutta import. - """ diff --git a/src/wuttasync/importing/__init__.py b/src/wuttasync/importing/__init__.py index 545cbb9..03a421f 100644 --- a/src/wuttasync/importing/__init__.py +++ b/src/wuttasync/importing/__init__.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar +# Copyright © 2024 Lance Edgar # # This file is part of Wutta Framework. # @@ -32,7 +32,7 @@ And some :term:`import handler` base classes: * :class:`~wuttasync.importing.handlers.ImportHandler` * :class:`~wuttasync.importing.handlers.FromFileHandler` * :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler` -* :class:`~wuttasync.importing.handlers.ToWuttaHandler` +* :class:`~wuttasync.importing.wutta.ToWuttaHandler` And some :term:`importer` base classes: @@ -42,12 +42,7 @@ And some :term:`importer` base classes: * :class:`~wuttasync.importing.model.ToWutta` """ -from .handlers import ( - Orientation, - ImportHandler, - FromFileHandler, - ToSqlalchemyHandler, - ToWuttaHandler, -) +from .handlers import Orientation, ImportHandler, FromFileHandler, ToSqlalchemyHandler from .base import Importer, FromFile, ToSqlalchemy from .model import ToWutta +from .wutta import ToWuttaHandler diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py index b625b17..629ead6 100644 --- a/src/wuttasync/importing/base.py +++ b/src/wuttasync/importing/base.py @@ -184,19 +184,6 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- :meth:`get_target_cache()`. """ - default_keys = None - """ - In certain edge cases, the importer class must declare its key - list without using :attr:`keys`. - - (As of now this only happens with - :class:`~wuttasync.importing.versions.FromWuttaToVersions` which - must dynamically create importer classes.) - - If applicable, this value is used as fallback for - :meth:`get_keys()`. - """ - max_create = None max_update = None max_delete = None @@ -336,54 +323,19 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- def get_keys(self): """ - Retrieve the list of key field(s) for use with import/export. - The result is cached, so the key list is only calculated once. - - Many importers have just one key field, but we always assume a - key *list* - so this often is a list with just one field. + Must return the key field(s) for use with import/export. All fields in this list should also be found in the output for :meth:`get_fields()`. - Many importers will declare this via :attr:`keys` (or - :attr:`key`) static attribute:: - - class SprocketImporter(Importer): - - # nb. all these examples work the same - - # 'keys' is the preferred attribute - keys = ("sprocket_id",) # <-- the "canonical" way - keys = ["sprocket_id"] - keys = "sprocket_id" - - # 'key' is not preferred, but works - key = ("sprocket_id",) - key = "sprocket_id" - - If neither ``keys`` nor ``key`` is set, as a special case - :attr:`default_keys` is used if set. - - If no keys were declared, the list is inspected from the model - class via - :func:`sqlalchemy-utils:sqlalchemy_utils.functions.get_primary_keys()`. - - In any case, the determination is made only once. This method - also *sets* :attr:`keys` on the instance, so it will return - that as-is for subsequent calls. - :returns: List of "key" field names. """ keys = None - # nb. prefer 'keys' but use 'key' as fallback if "keys" in self.__dict__: keys = self.__dict__["keys"] elif "key" in self.__dict__: keys = self.__dict__["key"] - else: - keys = self.default_keys - if keys: if isinstance(keys, str): keys = self.config.parse_list(keys) @@ -459,10 +411,7 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- updated = [] deleted = [] - model_title = self.get_model_title() - log.debug( - "using key fields for %s: %s", model_title, ", ".join(self.get_keys()) - ) + log.debug("using key fields: %s", ", ".join(self.get_keys())) # get complete set of normalized source data if source_data is None: @@ -471,7 +420,8 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public- # nb. prune duplicate records from source data source_data, source_keys = self.get_unique_data(source_data) - log.debug("got %s %s records from source", len(source_data), model_title) + model_title = self.get_model_title() + log.debug(f"got %s {model_title} records from source", len(source_data)) # maybe cache existing target data if self.caches_target: @@ -1321,139 +1271,10 @@ class FromFile(Importer): self.input_file.close() -class QueryWrapper: - """ - Simple wrapper for a SQLAlchemy query, to make it sort of behave - so that an importer can treat it as a data record list. - - :param query: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance - """ - - def __init__(self, query): - self.query = query - - def __len__(self): - try: - return len(self.query) - except TypeError: - return self.query.count() - - def __iter__(self): - return iter(self.query) - - -class FromSqlalchemy(Importer): # pylint: disable=abstract-method - """ - Base class for importer/exporter using SQL/ORM query as data - source. - - Subclass should define :attr:`source_model_class` in which case - the source query is automatic. And/or override - :meth:`get_source_query()` to customize. - - See also :class:`FromSqlalchemyMirror` and :class:`ToSqlalchemy`. - """ - - source_model_class = None - """ - Reference to the :term:`data model` class representing the source. - - This normally is a SQLAlchemy mapped class, e.g. - :class:`~wuttjamaican:wuttjamaican.db.model.base.Person` for - exporting from the Wutta People table. - """ - - source_session = None - """ - Reference to the open :term:`db session` for the data source. - - The importer must be given this reference when instantiated by the - :term:`import handler`. This is handled automatically if using - :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`. - """ - - def get_source_objects(self): - """ - This method is responsible for fetching "raw" (non-normalized) - records from data source. - - (See also the parent method docs for - :meth:`~wuttasync.importing.base.Importer.get_source_objects()`.) - - It calls :meth:`get_source_query()` and then wraps that in a - :class:`QueryWrapper`, which is then returned. - - Note that this method does not technically "retrieve" records - from the query; that happens automatically later. - - :returns: :class:`QueryWrapper` for the source query - """ - query = self.get_source_query() - return QueryWrapper(query) - - def get_source_query(self): - """ - This returns the SQL/ORM query used to fetch source - data. It is called from :meth:`get_source_objects()`. - - Default logic just makes a simple ``SELECT * FROM TABLE`` kind - of query. Subclass can override as needed. - - :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance - """ - return self.source_session.query(self.source_model_class) - - -class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method - """ - Special base class for when the source and target are effectively - mirrored, and can each be represented by the same :term:`data - model`. - - The assumption is that SQLAlchemy ORM is used on both sides, even - though this base class only defines the source side (it inherits - from :class:`FromSqlalchemy`). - - There are two main use cases for this: - - * sync between app nodes - * sync version tables - - When 2 app nodes are synced, the source and target are "the same" - in a schema sense, e.g. ``sprockets on node 01 => sprockets on - node 02``. - - When version tables are synced, the same schema can be used for - the "live" table and the "version" table, e.g. ``sprockets => - sprocket versions``. - """ - - @property - def source_model_class(self): - """ - This returns the :attr:`~Importer.model_class` since source - and target must share common schema. - """ - return self.model_class - - def normalize_source_object(self, obj): - """ - Since source/target share schema, there should be no tricky - normalization involved. - - This calls :meth:`~Importer.normalize_target_object()` since - that logic should already be defined. This ensures the same - normalization is used on both sides. - """ - return self.normalize_target_object(obj) - - class ToSqlalchemy(Importer): """ Base class for importer/exporter which uses SQLAlchemy ORM on the target side. - - See also :class:`FromSqlalchemy`. """ caches_target = True @@ -1491,8 +1312,6 @@ class ToSqlalchemy(Importer): Returns an ORM query suitable to fetch existing objects from the target side. This is called from :meth:`get_target_objects()`. - - :returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance """ return self.target_session.query(self.model_class) diff --git a/src/wuttasync/importing/csv.py b/src/wuttasync/importing/csv.py index 9190099..ab0bf21 100644 --- a/src/wuttasync/importing/csv.py +++ b/src/wuttasync/importing/csv.py @@ -38,7 +38,8 @@ from wuttjamaican.db.util import make_topo_sortkey, UUID from wuttjamaican.util import parse_bool from .base import FromFile -from .handlers import FromFileHandler, ToWuttaHandler +from .handlers import FromFileHandler +from .wutta import ToWuttaHandler from .model import ToWutta @@ -238,8 +239,6 @@ class FromCsvToSqlalchemyHandlerMixin: """ raise NotImplementedError - # TODO: pylint (correctly) flags this as duplicate code, matching - # on the wuttasync.importing.versions module - should fix? def define_importers(self): """ This mixin overrides typical (manual) importer definition, and @@ -253,7 +252,6 @@ class FromCsvToSqlalchemyHandlerMixin: importers = {} model = self.get_target_model() - # pylint: disable=duplicate-code # mostly try to make an importer for every data model for name in dir(model): cls = getattr(model, name) diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index ac13f28..e9c6ac3 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -26,14 +26,10 @@ Data Import / Export Handlers import logging import os -import sys from collections import OrderedDict from enum import Enum -import humanize - from wuttjamaican.app import GenericHandler -from wuttjamaican.diffs import Diff log = logging.getLogger(__name__) @@ -48,7 +44,7 @@ class Orientation(Enum): EXPORT = "export" -class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods +class ImportHandler(GenericHandler): """ Base class for all import/export handlers. @@ -125,47 +121,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods :meth:`commit_transaction()`. """ - process_started = None - - warnings = False - """ - Boolean indicating the import/export should run in "warnings" - mode. - - If set, this declares that no changes are expected for the - import/export job. If any changes do occur with this flag set, a - diff warning email is sent within :meth:`process_changes()`. - - See also :attr:`warnings_recipients`, - :attr:`warnings_max_diffs` and :attr:`warnings_email_key`. - """ - - warnings_email_key = None - """ - Explicit :term:`email key` for sending the diff warning email, - *unique to this import/export type*. - - Handlers do not normally set this, so the email key is determined - automatically within :meth:`get_warnings_email_key()`. - - See also :attr:`warnings`. - """ - - warnings_recipients = None - """ - Explicit recipient list for the warning email. If not set, the - recipients are determined automatically via config. - - See also :attr:`warnings`. - """ - - warnings_max_diffs = 15 - """ - Max number of record diffs (per model) to show in the warning email. - - See also :attr:`warnings`. - """ - importers = None """ This should be a dict of all importer/exporter classes available @@ -209,21 +164,18 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods @classmethod def get_key(cls): """ - Returns the :term:`import/export key` for the handler. This - is a combination of :attr:`source_key` and :attr:`target_key` - and :attr:`orientation`. + Returns the "full key" for the handler. This is a combination + of :attr:`source_key` and :attr:`target_key` and + :attr:`orientation`. - For instance in the case of Wutta → CSV export, the key is: - ``export.to_csv.from_wutta`` + For instance in the case of CSV → Wutta, the full handler key + is ``to_wutta.from_csv.import``. - Note that more than one handler may use the same key; but only - one will be configured as the "designated" handler for that - key, a la - :meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`. - - See also :meth:`get_spec()`. + Note that more than one handler may return the same full key + here; but only one will be configured as the "default" handler + for that key. See also :meth:`get_spec()`. """ - return f"{cls.orientation.value}.to_{cls.target_key}.from_{cls.source_key}" + return f"to_{cls.target_key}.from_{cls.source_key}.{cls.orientation.value}" @classmethod def get_spec(cls): @@ -257,20 +209,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods """ Returns the display title for the data source. - By default this returns :attr:`source_key`, but this can be - overriden by class attribute. - - Base class can define ``generic_source_title`` to provide a - new default:: - - class FromExcelHandler(ImportHandler): - generic_source_title = "Excel File" - - Subclass can define ``source_title`` to be explicit:: - - class FromExcelToWutta(FromExcelHandler, ToWuttaHandler): - source_title = "My Spreadsheet" - See also :meth:`get_title()` and :meth:`get_target_title()`. """ if hasattr(self, "source_title"): @@ -283,20 +221,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods """ Returns the display title for the data target. - By default this returns :attr:`target_key`, but this can be - overriden by class attribute. - - Base class can define ``generic_target_title`` to provide a - new default:: - - class ToExcelHandler(ImportHandler): - generic_target_title = "Excel File" - - Subclass can define ``target_title`` to be explicit:: - - class FromWuttaToExcel(FromWuttaHandler, ToExcelHandler): - target_title = "My Spreadsheet" - See also :meth:`get_title()` and :meth:`get_source_title()`. """ if hasattr(self, "target_title"): @@ -326,17 +250,11 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods * :meth:`begin_transaction()` * :meth:`get_importer()` * :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter) - * :meth:`process_changes()` * :meth:`rollback_transaction()` * :meth:`commit_transaction()` """ kwargs = self.consume_kwargs(kwargs) - self.process_started = self.app.localtime() self.begin_transaction() - changes = OrderedDict() - - if not keys: - keys = list(self.importers) success = False try: @@ -347,31 +265,22 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods # invoke importer importer = self.get_importer(key, **kwargs) created, updated, deleted = importer.process_data() - changed = bool(created or updated or deleted) # log what happened msg = "%s: added %d; updated %d; deleted %d %s records" if self.dry_run: msg += " (dry run)" - logger = log.warning if changed and self.warnings else log.info - logger( + log.info( msg, self.get_title(), len(created), len(updated), len(deleted), key ) - # keep track of any changes - if changed: - changes[key] = created, updated, deleted - - # post-processing for all changes - if changes: - self.process_changes(changes) - - success = True - except: log.exception("what should happen here?") # TODO raise + else: + success = True + finally: if not success: log.warning("something failed, so transaction was rolled back") @@ -405,17 +314,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods if "dry_run" in kwargs: self.dry_run = kwargs["dry_run"] - if "warnings" in kwargs: - self.warnings = kwargs.pop("warnings") - - if "warnings_recipients" in kwargs: - self.warnings_recipients = self.config.parse_list( - kwargs.pop("warnings_recipients") - ) - - if "warnings_max_diffs" in kwargs: - self.warnings_max_diffs = kwargs.pop("warnings_max_diffs") - return kwargs def begin_transaction(self): @@ -614,113 +512,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods """ return kwargs - def process_changes(self, changes): - """ - Run post-processing operations on the given changes, if - applicable. - - This method is called by :meth:`process_data()`, if any - changes were made. - - Default logic will send a "diff warning" email to the - configured recipient(s), if :attr:`warnings` mode is enabled. - If it is not enabled, nothing happens. - - :param changes: :class:`~python:collections.OrderedDict` of - changes from the overall import/export job. The structure - is described below. - - Keys for the ``changes`` dict will be model/importer names, - for instance:: - - { - "Sprocket": {...}, - "User": {...}, - } - - Value for each model key is a 3-tuple of ``(created, updated, - deleted)``. Each of those elements is a list:: - - { - "Sprocket": ( - [...], # created - [...], # updated - [...], # deleted - ), - } - - The list elements are always tuples, but the structure - varies:: - - { - "Sprocket": ( - [ # created, 2-tuples - (obj, source_data), - ], - [ # updated, 3-tuples - (obj, source_data, target_data), - ], - [ # deleted, 2-tuples - (obj, target_data), - ], - ), - } - """ - if not self.warnings: - return - - def make_diff(*args, **kwargs): - return Diff(self.config, *args, **kwargs) - - runtime = self.app.localtime() - self.process_started - data = { - "handler": self, - "title": self.get_title(), - "source_title": self.get_source_title(), - "target_title": self.get_target_title(), - "dry_run": self.dry_run, - "argv": sys.argv, - "runtime": runtime, - "runtime_display": humanize.naturaldelta(runtime), - "changes": changes, - "make_diff": make_diff, - "max_diffs": self.warnings_max_diffs, - } - - # maybe override recipients - kw = {} - if self.warnings_recipients: - kw["to"] = self.warnings_recipients - # TODO: should we in fact clear these..? - kw["cc"] = [] - kw["bcc"] = [] - - # send the email - email_key = self.get_warnings_email_key() - self.app.send_email(email_key, data, fallback_key="import_export_warning", **kw) - - log.info("%s: warning email was sent", self.get_title()) - - def get_warnings_email_key(self): - """ - Returns the :term:`email key` to be used for sending the diff - warning email. - - The email key should be unique to this import/export type - (really, the :term:`import/export key`) but not necessarily - unique to one handler. - - If :attr:`warnings_email_key` is set, it will be used as-is. - - Otherwise one is generated from :meth:`get_key()`. - - :returns: Email key for diff warnings - """ - if self.warnings_email_key: - return self.warnings_email_key - - return self.get_key().replace(".", "_") + "_warning" - class FromFileHandler(ImportHandler): """ @@ -747,129 +538,9 @@ class FromFileHandler(ImportHandler): super().process_data(*keys, **kwargs) -class FromSqlalchemyHandler(ImportHandler): - """ - Base class for import/export handlers using SQLAlchemy ORM (DB) as - data source. - - This is meant to be used with importers/exporters which inherit - from :class:`~wuttasync.importing.base.FromSqlalchemy`. It will - set the - :attr:`~wuttasync.importing.base.FromSqlalchemy.source_session` - attribute when making them; cf. :meth:`get_importer_kwargs()`. - - This is the base class for :class:`FromWuttaHandler`, but can be - used with any database. - - See also :class:`ToSqlalchemyHandler`. - """ - - source_session = None - """ - Reference to the :term:`db session` for data source. - - This will be ``None`` unless a transaction is running. - """ - - def begin_source_transaction(self): - """ - This calls :meth:`make_source_session()` and assigns the - result to :attr:`source_session`. - """ - self.source_session = self.make_source_session() - - def commit_source_transaction(self): - """ - This commits and closes :attr:`source_session`. - """ - self.source_session.commit() - self.source_session.close() - self.source_session = None - - def rollback_source_transaction(self): - """ - This rolls back, then closes :attr:`source_session`. - """ - self.source_session.rollback() - self.source_session.close() - self.source_session = None - - def make_source_session(self): - """ - Make and return a new :term:`db session` for the data source. - - Default logic is not implemented; subclass must override. - - :returns: :class:`~sqlalchemy.orm.Session` instance - """ - raise NotImplementedError - - def get_importer_kwargs(self, key, **kwargs): - """ - This modifies the new importer kwargs to add: - - * ``source_session`` - reference to :attr:`source_session` - - See also docs for parent method, - :meth:`~ImportHandler.get_importer_kwargs()`. - """ - kwargs = super().get_importer_kwargs(key, **kwargs) - kwargs["source_session"] = self.source_session - return kwargs - - -class FromWuttaHandler(FromSqlalchemyHandler): - """ - Handler for import/export which uses Wutta ORM (:term:`app - database`) as data source. - - This inherits from :class:`FromSqlalchemyHandler`. - - See also :class:`ToWuttaHandler`. - """ - - source_key = "wutta" - "" # nb. suppress docs - - def get_source_title(self): - """ - This overrides default logic to use - :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.get_title()` - as the default value. - - Subclass can still define - :attr:`~wuttasync.importing.handlers.ImportHandler.source_title` - (or - :attr:`~wuttasync.importing.handlers.ImportHandler.generic_source_title`) - to customize. - - See also docs for parent method: - :meth:`~wuttasync.importing.handlers.ImportHandler.get_source_title()` - """ - if hasattr(self, "source_title"): - return self.source_title - if hasattr(self, "generic_source_title"): - return self.generic_source_title - return self.app.get_title() - - def make_source_session(self): - """ - This calls - :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` - and returns it. - """ - return self.app.make_session() - - class ToSqlalchemyHandler(ImportHandler): """ - Base class for import/export handlers which target a SQLAlchemy - ORM (DB). - - This is the base class for :class:`ToWuttaHandler`, but can be - used with any database. - - See also :class:`FromSqlalchemyHandler`. + Handler for import/export which targets a SQLAlchemy ORM (DB). """ target_session = None @@ -920,37 +591,3 @@ class ToSqlalchemyHandler(ImportHandler): kwargs = super().get_importer_kwargs(key, **kwargs) kwargs.setdefault("target_session", self.target_session) return kwargs - - -class ToWuttaHandler(ToSqlalchemyHandler): - """ - Handler for import/export which targets Wutta ORM (:term:`app - database`). - - This inherits from :class:`ToSqlalchemyHandler`. - - See also :class:`FromWuttaHandler`. - """ - - target_key = "wutta" - "" # nb. suppress docs - - def get_target_title(self): # pylint: disable=empty-docstring - """ """ - # nb. we override parent to use app title as default - if hasattr(self, "target_title"): - return self.target_title - if hasattr(self, "generic_target_title"): - return self.generic_target_title - return self.app.get_title() - - def make_target_session(self): - """ - Call - :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` - and return the result. - - :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session` - instance. - """ - return self.app.make_session() diff --git a/src/wuttasync/importing/versions.py b/src/wuttasync/importing/versions.py deleted file mode 100644 index cda77c9..0000000 --- a/src/wuttasync/importing/versions.py +++ /dev/null @@ -1,353 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -Importing Versions - -This is a special type of import, only relevant when data versioning -is enabled. - -See the handler class for more info: :class:`FromWuttaToVersions` -""" - -from collections import OrderedDict - -from sqlalchemy_utils.functions import get_primary_keys - -from wuttjamaican.db.util import make_topo_sortkey - -from .handlers import FromWuttaHandler, ToWuttaHandler -from .wutta import FromWuttaMirror -from .model import ToWutta - - -class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler): - """ - Handler for Wutta -> Versions import. - - The purpose of this is to ensure version tables accurately reflect - the current "live" data set, for given table(s). It is only - relevant/usable if versioning is configured and enabled. For more - on that see :doc:`wutta-continuum:index`. - - For a given import model, the source is the "live" table, target - is the "version" table - both in the same :term:`app database`. - - When reading data from the target side, it only grabs the "latest" - (valid) version record for each comparison to source. - - When changes are needed, instead of updating the existing version - record, it always writes a new version record. - - This handler will dynamically create importers for all versioned - models in the :term:`app model`; see - :meth:`make_importer_factory()`. - """ - - target_key = "versions" - target_title = "Versions" - - continuum_uow = None - """ - Reference to the - :class:`sqlalchemy-continuum:`sqlalchemy_continuum.UnitOfWork` - created (by the SQLAlchemy-Continuum ``versioning_manager``) when - the transaction begins. - - See also :attr:`continuum_txn` and - :meth:`begin_target_transaction()`. - """ - - continuum_txn = None - """ - Reference to the SQLAlchemy-Continuum ``transaction`` record, to - which any new version records will associate (if needed). - - This transaction will track the effective user responsible for - the change(s), their client IP, and timestamp. - - This reference is passed along to the importers as well (as - :attr:`~FromWuttaToVersionBase.continuum_txn`) via - :meth:`get_importer_kwargs()`. - - See also :attr:`continuum_uow`. - """ - - continuum_comment = None - - def consume_kwargs(self, kwargs): - kwargs = super().consume_kwargs(kwargs) - self.continuum_comment = kwargs.pop("comment", None) - return kwargs - - def begin_target_transaction(self): - # pylint: disable=line-too-long - """ - In addition to normal logic, this does some setup for - SQLAlchemy-Continuum: - - It establishes a "unit of work" by calling - :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.VersioningManager.unit_of_work()`, - assigning the result to :attr:`continuum_uow`. - - It then calls - :meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()` - and assigns that to :attr:`continuum_txn`. - - It also sets the comment for the transaction, if applicable. - - See also docs for parent method: - :meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()` - """ - import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel - - super().begin_target_transaction() - - self.continuum_uow = continuum.versioning_manager.unit_of_work( - self.target_session - ) - - self.continuum_txn = self.continuum_uow.create_transaction(self.target_session) - - if self.continuum_comment: - self.continuum_txn.meta = {"comment": self.continuum_comment} - - def get_importer_kwargs(self, key, **kwargs): - """ - This modifies the new importer kwargs to add: - - * ``continuum_txn`` - reference to :attr:`continuum_txn` - - See also docs for parent method: - :meth:`~wuttasync.importing.handlers.ImportHandler.get_importer_kwargs()` - """ - kwargs = super().get_importer_kwargs(key, **kwargs) - kwargs["continuum_txn"] = self.continuum_txn - return kwargs - - # TODO: pylint (correctly) flags this as duplicate code, matching - # on the wuttasync.importing.csv module - should fix? - def define_importers(self): - """ - This overrides typical (manual) importer definition, instead - generating importers for all versioned models. - - It will inspect the :term:`app model` and call - :meth:`make_importer_factory()` for each model found, keeping - only the valid importers. - - See also the docs for parent method: - :meth:`~wuttasync.importing.handlers.ImportHandler.define_importers()` - """ - model = self.app.model - importers = {} - - # pylint: disable=duplicate-code - # mostly try to make an importer for every data model - for name in dir(model): - cls = getattr(model, name) - if ( - isinstance(cls, type) - and issubclass(cls, model.Base) - and cls is not model.Base - ): - # only keep "good" importers, i.e. for versioned models - if factory := self.make_importer_factory(cls, name): - importers[name] = factory - - # sort importers according to schema topography - topo_sortkey = make_topo_sortkey(model) - importers = OrderedDict( - [(name, importers[name]) for name in sorted(importers, key=topo_sortkey)] - ) - - return importers - - def make_importer_factory(self, model_class, name): - """ - Try to generate a new :term:`importer` class for the given - :term:`data model`. This is called by - :meth:`define_importers()`. - - If the provided ``model_class`` is not versioned, this will - fail and return ``None``. - - For a versioned model, the new importer class will inherit - from :class:`FromWuttaToVersionBase`. - - Its (target) - :attr:`~wuttasync.importing.base.Importer.model_class` will be - set to the **version** model. - - Its - :attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class` - will be set to the **normal** model. - - :param model_class: A (normal, not version) data model class. - - :param name: The "model name" for the importer. New class - name will be based on this, so e.g. ``Widget`` model name - becomes ``WidgetImporter`` class name. - - :returns: The new class, or ``None`` - """ - import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel - - try: - version_class = continuum.version_class(model_class) - except continuum.exc.ClassNotVersioned: - return None - - return type( - f"{name}Importer", - (FromWuttaToVersionBase,), - { - "source_model_class": model_class, - "model_class": version_class, - "default_keys": list(get_primary_keys(model_class)), - }, - ) - - -class FromWuttaToVersionBase(FromWuttaMirror, ToWutta): - """ - Base importer class for Wutta -> Versions. - - This imports from - :class:`~wuttasync.importing.wutta.FromWuttaMirror` and - :class:`~wuttasync.importing.model.ToWutta`. - - The import handler will dynamically generate importers using this - base class; see - :meth:`~FromWuttaToVersions.make_importer_factory()`. - """ - - continuum_txn = None - """ - Reference to the handler's attribute of the same name: - :attr:`~FromWuttaToVersions.continuum_txn` - - This is the SQLAlchemy-Continuum ``transaction`` record, to which - any new version records will associate (if needed). - - This transaction will track the effective user responsible for - the change(s), their client IP, and timestamp. - """ - - def get_simple_fields(self): # pylint: disable=empty-docstring - """ """ - fields = super().get_simple_fields() - unwanted = ["transaction_id", "operation_type", "end_transaction_id"] - fields = [field for field in fields if field not in unwanted] - return fields - - def get_target_query(self, source_data=None): - """ - This modifies the normal query to ensure we only get the - "latest valid" version for each record, for comparison to - source. - - .. note:: - - In some cases, it still may be possible for multiple - "latest" versions to match for a given record. This means - inconsistent data; a warning should be logged if so, and - you must track it down... - - See also docs for parent method: - :meth:`~wuttasync.importing.base.ToSqlalchemy.get_target_query()` - """ - import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel - - # pylint: disable=singleton-comparison - return ( - self.target_session.query(self.model_class) - .filter(self.model_class.end_transaction_id == None) - .filter(self.model_class.operation_type != continuum.Operation.DELETE) - ) - - def normalize_target_object(self, obj): # pylint: disable=empty-docstring - """ """ - data = super().normalize_target_object(obj) - - # we want to add the original version object to normalized - # data, so we can access it later for updating if needed. but - # this method is called for *both* sides (source+target) since - # this is a "mirrored" importer. so we must check the type - # and only cache true versions, ignore "normal" objects. - if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type - obj, self.model_class - ): - data["_objref"] = obj - - return data - - def make_version( # pylint: disable=missing-function-docstring - self, source_data, operation_type - ): - key = self.get_record_key(source_data) - with self.target_session.no_autoflush: - version = self.make_empty_object(key) - self.populate(version, source_data) - version.transaction = self.continuum_txn - version.operation_type = operation_type - self.target_session.add(version) - return version - - def populate(self, obj, data): # pylint: disable=missing-function-docstring - keys = self.get_keys() - for field in self.get_simple_fields(): - if field not in keys and field in data and field in self.fields: - setattr(obj, field, data[field]) - - def create_target_object(self, key, source_data): # pylint: disable=empty-docstring - """ """ - import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel - - return self.make_version(source_data, continuum.Operation.INSERT) - - def update_target_object( # pylint: disable=empty-docstring - self, obj, source_data, target_data=None - ): - """ """ - import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel - - # when we "update" it always involves making a *new* version - # record. but that requires actually updating the "previous" - # version to indicate the new version's transaction. - prev_version = target_data.pop("_objref") - prev_version.end_transaction_id = self.continuum_txn.id - - return self.make_version(source_data, continuum.Operation.UPDATE) - - def delete_target_object(self, obj): # pylint: disable=empty-docstring - """ """ - import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel - - # nb. `obj` here is the existing/old version record; we update - # it to indicate the new version's transaction. - obj.end_transaction_id = self.continuum_txn.id - - # add new "DELETE" version record. values should be the same as - # for "previous" (existing/old) version. - source_data = self.normalize_target_object(obj) - return self.make_version(source_data, continuum.Operation.DELETE) diff --git a/src/wuttasync/importing/wutta.py b/src/wuttasync/importing/wutta.py index 882f7df..9de4822 100644 --- a/src/wuttasync/importing/wutta.py +++ b/src/wuttasync/importing/wutta.py @@ -21,13 +21,37 @@ # ################################################################################ """ -Wutta → Wutta import/export +Wutta ⇄ Wutta import/export """ -from .base import FromSqlalchemyMirror +from .handlers import ToSqlalchemyHandler -class FromWuttaMirror(FromSqlalchemyMirror): # pylint: disable=abstract-method +class ToWuttaHandler(ToSqlalchemyHandler): """ - Base class for Wutta -> Wutta data importers. + Handler for import/export which targets Wutta ORM (:term:`app + database`). """ + + target_key = "wutta" + "" # nb. suppress docs + + def get_target_title(self): # pylint: disable=empty-docstring + """ """ + # nb. we override parent to use app title as default + if hasattr(self, "target_title"): + return self.target_title + if hasattr(self, "generic_target_title"): + return self.generic_target_title + return self.app.get_title() + + def make_target_session(self): + """ + Call + :meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()` + and return the result. + + :returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session` + instance. + """ + return self.app.make_session() diff --git a/src/wuttasync/testing.py b/src/wuttasync/testing.py deleted file mode 100644 index 1daad1f..0000000 --- a/src/wuttasync/testing.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8; -*- -################################################################################ -# -# WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024-2025 Lance Edgar -# -# This file is part of Wutta Framework. -# -# Wutta Framework is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) any -# later version. -# -# Wutta Framework is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# Wutta Framework. If not, see . -# -################################################################################ -""" -Testing utilities -""" - -from wuttjamaican.testing import ConfigTestCase - - -class ImportExportWarningTestCase(ConfigTestCase): - """ - Base class for testing the import/export warning email settings. - - This inherits from - :class:`~wuttjamaican:wuttjamaican.testing.ConfigTestCase`. - - Example usage:: - - from wuttasync.testing import ImportExportWarningTestCase - - class TestEmailSettings(ImportExportWarningTestCase): - - def test_import_to_wutta_from_foo_warning(self): - self.do_test_preview("import_to_wutta_from_foo_warning") - - def test_export_to_foo_from_wutta_warning(self): - self.do_test_preview("export_to_foo_from_wutta_warning") - """ - - app_title = "Wutta Poser" - - def setUp(self): - self.setup_config() - self.config.setdefault("wutta.app_title", self.app_title) - - def make_preview( # pylint: disable=missing-function-docstring,unused-argument - self, key, mode="html" - ): - handler = self.app.get_email_handler() - setting = handler.get_email_setting(key) - context = setting.sample_data() - return handler.get_auto_html_body( - setting.key, context, fallback_key=setting.fallback_key - ) - - def do_test_preview(self, key): # pylint: disable=missing-function-docstring - body = self.make_preview(key, mode="html") - self.assertIn("Diff warning for ", body) diff --git a/tests/cli/test_base.py b/tests/cli/test_base.py index b8fc954..991358e 100644 --- a/tests/cli/test_base.py +++ b/tests/cli/test_base.py @@ -34,10 +34,6 @@ class TestImportCommandHandler(DataTestCase): handler = self.make_handler(import_handler=myhandler) self.assertIs(handler.import_handler, myhandler) - # as key - handler = self.make_handler(key="import.to_wutta.from_csv") - self.assertIsInstance(handler.import_handler, FromCsvToWutta) - def test_run(self): handler = self.make_handler( import_handler="wuttasync.importing.csv:FromCsvToWutta" diff --git a/tests/cli/test_import_versions.py b/tests/cli/test_import_versions.py deleted file mode 100644 index ea1617d..0000000 --- a/tests/cli/test_import_versions.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8; -*- - -from unittest import TestCase -from unittest.mock import MagicMock, patch - -from wuttasync.cli import import_versions as mod, ImportCommandHandler - - -class TestImportCsv(TestCase): - - def test_basic(self): - params = { - "models": [], - "create": True, - "update": True, - "delete": False, - "dry_run": True, - } - ctx = MagicMock(params=params) - with patch.object(ImportCommandHandler, "run") as run: - mod.import_versions(ctx) - run.assert_called_once_with(params) diff --git a/tests/importing/test_base.py b/tests/importing/test_base.py index 9f83ae3..08c37a2 100644 --- a/tests/importing/test_base.py +++ b/tests/importing/test_base.py @@ -2,8 +2,6 @@ from unittest.mock import patch -from sqlalchemy import orm - from wuttjamaican.testing import DataTestCase from wuttasync.importing import base as mod, ImportHandler, Orientation @@ -80,31 +78,13 @@ class TestImporter(DataTestCase): def test_get_keys(self): model = self.app.model - - # nb. get_keys() will cache the return value, so must - # re-create importer for each test - - # keys inspected from model by default imp = self.make_importer(model_class=model.Setting) self.assertEqual(imp.get_keys(), ["name"]) - imp = self.make_importer(model_class=model.User) - self.assertEqual(imp.get_keys(), ["uuid"]) - - # class may define 'keys' - imp = self.make_importer(model_class=model.User) - with patch.object(imp, "keys", new=["foo", "bar"], create=True): + with patch.multiple(imp, create=True, key="value"): + self.assertEqual(imp.get_keys(), ["value"]) + with patch.multiple(imp, create=True, keys=["foo", "bar"]): self.assertEqual(imp.get_keys(), ["foo", "bar"]) - # class may define 'key' - imp = self.make_importer(model_class=model.User) - with patch.object(imp, "key", new="whatever", create=True): - self.assertEqual(imp.get_keys(), ["whatever"]) - - # class may define 'default_keys' - imp = self.make_importer(model_class=model.User) - with patch.object(imp, "default_keys", new=["baz", "foo"]): - self.assertEqual(imp.get_keys(), ["baz", "foo"]) - def test_process_data(self): model = self.app.model imp = self.make_importer( @@ -671,106 +651,6 @@ class TestFromFile(DataTestCase): close.assert_called_once_with() -class TestQueryWrapper(DataTestCase): - - def test_basic(self): - model = self.app.model - - p1 = model.Person(full_name="John Doe") - self.session.add(p1) - p2 = model.Person(full_name="Jane Doe") - self.session.add(p2) - self.session.commit() - - # cannot get count via len(query), must use query.count() - query = self.session.query(model.Person) - self.assertEqual(query.count(), 2) - self.assertRaises(TypeError, len, query) - - # but can use len(wrapper) - wrapper = mod.QueryWrapper(query) - self.assertEqual(len(wrapper), 2) - - # iter(wrapper) should work too - people = [p for p in wrapper] - self.assertEqual(people, [p1, p2]) - people = [p for p in iter(wrapper)] - self.assertEqual(people, [p1, p2]) - people = [p for p in list(wrapper)] - self.assertEqual(people, [p1, p2]) - - -class TestFromSqlalchemy(DataTestCase): - - def setUp(self): - self.setup_db() - self.handler = ImportHandler(self.config) - - def make_importer(self, **kwargs): - kwargs.setdefault("handler", self.handler) - return mod.FromSqlalchemy(self.config, **kwargs) - - def test_get_source_query(self): - model = self.app.model - imp = self.make_importer( - source_model_class=model.Upgrade, source_session=self.session - ) - query = imp.get_source_query() - self.assertIsInstance(query, orm.Query) - froms = query.selectable.get_final_froms() - self.assertEqual(len(froms), 1) - table = froms[0] - self.assertEqual(table.name, "upgrade") - - def test_get_source_objects(self): - model = self.app.model - - user1 = model.User(username="fred") - self.session.add(user1) - user2 = model.User(username="bettie") - self.session.add(user2) - self.session.commit() - - imp = self.make_importer( - source_model_class=model.User, source_session=self.session - ) - result = imp.get_source_objects() - self.assertIsInstance(result, mod.QueryWrapper) - self.assertEqual(len(result), 2) - self.assertEqual(list(result), [user1, user2]) - - -class TestFromSqlalchemyMirror(DataTestCase): - - def setUp(self): - self.setup_db() - self.handler = ImportHandler(self.config) - - def make_importer(self, **kwargs): - kwargs.setdefault("handler", self.handler) - return mod.FromSqlalchemyMirror(self.config, **kwargs) - - def test_source_model_class(self): - model = self.app.model - - # source_model_class will mirror model_class - imp = self.make_importer(model_class=model.Upgrade) - self.assertIs(imp.model_class, model.Upgrade) - self.assertIs(imp.source_model_class, model.Upgrade) - - def test_normalize_source_object(self): - model = self.app.model - imp = self.make_importer(model_class=model.Upgrade) - upgrade = model.Upgrade() - - # normalize_source_object() should invoke normalize_target_object() - with patch.object(imp, "normalize_target_object") as normalize_target_object: - normalize_target_object.return_value = 42 - result = imp.normalize_source_object(upgrade) - self.assertEqual(result, 42) - normalize_target_object.assert_called_once_with(upgrade) - - class TestToSqlalchemy(DataTestCase): def setUp(self): diff --git a/tests/importing/test_handlers.py b/tests/importing/test_handlers.py index c01b405..9bd0157 100644 --- a/tests/importing/test_handlers.py +++ b/tests/importing/test_handlers.py @@ -2,18 +2,12 @@ from collections import OrderedDict from unittest.mock import patch -from uuid import UUID from wuttjamaican.testing import DataTestCase from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy -class FromFooToBar(mod.ImportHandler): - source_key = "foo" - target_key = "bar" - - class TestImportHandler(DataTestCase): def make_handler(self, **kwargs): @@ -36,10 +30,10 @@ class TestImportHandler(DataTestCase): def test_get_key(self): handler = self.make_handler() - self.assertEqual(handler.get_key(), "import.to_None.from_None") + self.assertEqual(handler.get_key(), "to_None.from_None.import") with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"): - self.assertEqual(handler.get_key(), "import.to_wutta.from_csv") + self.assertEqual(handler.get_key(), "to_wutta.from_csv.import") def test_get_spec(self): handler = self.make_handler() @@ -155,41 +149,15 @@ class TestImportHandler(DataTestCase): kw = {} result = handler.consume_kwargs(kw) self.assertIs(result, kw) - self.assertEqual(result, {}) - # dry_run (not consumed) + # captures dry-run flag self.assertFalse(handler.dry_run) kw["dry_run"] = True result = handler.consume_kwargs(kw) self.assertIs(result, kw) - self.assertIn("dry_run", kw) self.assertTrue(kw["dry_run"]) self.assertTrue(handler.dry_run) - # warnings (consumed) - self.assertFalse(handler.warnings) - kw["warnings"] = True - result = handler.consume_kwargs(kw) - self.assertIs(result, kw) - self.assertNotIn("warnings", kw) - self.assertTrue(handler.warnings) - - # warnings_recipients (consumed) - self.assertIsNone(handler.warnings_recipients) - kw["warnings_recipients"] = "bob@example.com" - result = handler.consume_kwargs(kw) - self.assertIs(result, kw) - self.assertNotIn("warnings_recipients", kw) - self.assertEqual(handler.warnings_recipients, ["bob@example.com"]) - - # warnings_max_diffs (consumed) - self.assertEqual(handler.warnings_max_diffs, 15) - kw["warnings_max_diffs"] = 30 - result = handler.consume_kwargs(kw) - self.assertIs(result, kw) - self.assertNotIn("warnings_max_diffs", kw) - self.assertEqual(handler.warnings_max_diffs, 30) - def test_define_importers(self): handler = self.make_handler() importers = handler.define_importers() @@ -219,94 +187,6 @@ class TestImportHandler(DataTestCase): KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting ) - def test_get_warnings_email_key(self): - handler = FromFooToBar(self.config) - - # default - key = handler.get_warnings_email_key() - self.assertEqual(key, "import_to_bar_from_foo_warning") - - # override - handler.warnings_email_key = "from_foo_to_bar" - key = handler.get_warnings_email_key() - self.assertEqual(key, "from_foo_to_bar") - - def test_process_changes(self): - model = self.app.model - handler = self.make_handler() - email_handler = self.app.get_email_handler() - - handler.process_started = self.app.localtime() - - alice = model.User(username="alice") - bob = model.User(username="bob") - charlie = model.User(username="charlie") - - changes = { - "User": ( - [ - ( - alice, - { - "uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"), - "username": "alice", - }, - ), - ], - [ - ( - bob, - { - "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), - "username": "bob", - }, - { - "uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"), - "username": "bobbie", - }, - ), - ], - [ - ( - charlie, - { - "uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"), - "username": "charlie", - }, - ), - ], - ), - } - - # no email if not in warnings mode - self.assertFalse(handler.warnings) - with patch.object(self.app, "send_email") as send_email: - handler.process_changes(changes) - send_email.assert_not_called() - - # email sent (to default recip) if in warnings mode - handler.warnings = True - self.config.setdefault("wutta.email.default.to", "admin@example.com") - with patch.object(email_handler, "deliver_message") as deliver_message: - handler.process_changes(changes) - deliver_message.assert_called_once() - args, kwargs = deliver_message.call_args - self.assertEqual(kwargs, {"recips": None}) - self.assertEqual(len(args), 1) - msg = args[0] - self.assertEqual(msg.to, ["admin@example.com"]) - - # can override email recip - handler.warnings_recipients = ["bob@example.com"] - with patch.object(email_handler, "deliver_message") as deliver_message: - handler.process_changes(changes) - deliver_message.assert_called_once() - args, kwargs = deliver_message.call_args - self.assertEqual(kwargs, {"recips": None}) - self.assertEqual(len(args), 1) - msg = args[0] - self.assertEqual(msg.to, ["bob@example.com"]) - class TestFromFileHandler(DataTestCase): @@ -333,97 +213,6 @@ class TestFromFileHandler(DataTestCase): process_data.assert_called_once_with(input_file_dir=self.tempdir) -class TestFromSqlalchemyHandler(DataTestCase): - - def make_handler(self, **kwargs): - return mod.FromSqlalchemyHandler(self.config, **kwargs) - - def test_make_source_session(self): - handler = self.make_handler() - self.assertRaises(NotImplementedError, handler.make_source_session) - - def test_begin_source_transaction(self): - handler = self.make_handler() - self.assertIsNone(handler.source_session) - with patch.object(handler, "make_source_session", return_value=self.session): - handler.begin_source_transaction() - self.assertIs(handler.source_session, self.session) - - def test_commit_source_transaction(self): - model = self.app.model - handler = self.make_handler() - handler.source_session = self.session - self.assertEqual(self.session.query(model.User).count(), 0) - - # nb. do not commit this yet - user = model.User(username="fred") - self.session.add(user) - - self.assertTrue(self.session.in_transaction()) - self.assertIn(user, self.session) - handler.commit_source_transaction() - self.assertIsNone(handler.source_session) - self.assertFalse(self.session.in_transaction()) - self.assertNotIn(user, self.session) # hm, surprising? - self.assertEqual(self.session.query(model.User).count(), 1) - - def test_rollback_source_transaction(self): - model = self.app.model - handler = self.make_handler() - handler.source_session = self.session - self.assertEqual(self.session.query(model.User).count(), 0) - - # nb. do not commit this yet - user = model.User(username="fred") - self.session.add(user) - - self.assertTrue(self.session.in_transaction()) - self.assertIn(user, self.session) - handler.rollback_source_transaction() - self.assertIsNone(handler.source_session) - self.assertFalse(self.session.in_transaction()) - self.assertNotIn(user, self.session) - self.assertEqual(self.session.query(model.User).count(), 0) - - def test_get_importer_kwargs(self): - handler = self.make_handler() - handler.source_session = self.session - kw = handler.get_importer_kwargs("User") - self.assertIn("source_session", kw) - self.assertIs(kw["source_session"], self.session) - - -class TestFromWuttaHandler(DataTestCase): - - def make_handler(self, **kwargs): - return mod.FromWuttaHandler(self.config, **kwargs) - - def test_get_source_title(self): - handler = self.make_handler() - - # uses app title by default - self.config.setdefault("wutta.app_title", "What About This") - self.assertEqual(handler.get_source_title(), "What About This") - - # or generic default if present - handler.generic_source_title = "WHATABOUTTHIS" - self.assertEqual(handler.get_source_title(), "WHATABOUTTHIS") - - # but prefer specific title if present - handler.source_title = "what_about_this" - self.assertEqual(handler.get_source_title(), "what_about_this") - - def test_make_source_session(self): - handler = self.make_handler() - - # makes "new" (mocked in our case) app session - with patch.object(self.app, "make_session") as make_session: - make_session.return_value = self.session - session = handler.make_source_session() - make_session.assert_called_once_with() - self.assertIs(session, self.session) - - class TestToSqlalchemyHandler(DataTestCase): def make_handler(self, **kwargs): @@ -467,34 +256,3 @@ class TestToSqlalchemyHandler(DataTestCase): kw = handler.get_importer_kwargs("Setting") self.assertIn("target_session", kw) self.assertIs(kw["target_session"], self.session) - - -class TestToWuttaHandler(DataTestCase): - - def make_handler(self, **kwargs): - return mod.ToWuttaHandler(self.config, **kwargs) - - def test_get_target_title(self): - handler = self.make_handler() - - # uses app title by default - self.config.setdefault("wutta.app_title", "What About This") - self.assertEqual(handler.get_target_title(), "What About This") - - # or generic default if present - handler.generic_target_title = "WHATABOUTTHIS" - self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS") - - # but prefer specific title if present - handler.target_title = "what_about_this" - self.assertEqual(handler.get_target_title(), "what_about_this") - - def test_make_target_session(self): - handler = self.make_handler() - - # makes "new" (mocked in our case) app session - with patch.object(self.app, "make_session") as make_session: - make_session.return_value = self.session - session = handler.make_target_session() - make_session.assert_called_once_with() - self.assertIs(session, self.session) diff --git a/tests/importing/test_versions.py b/tests/importing/test_versions.py deleted file mode 100644 index 2cd4ec0..0000000 --- a/tests/importing/test_versions.py +++ /dev/null @@ -1,262 +0,0 @@ -# -*- coding: utf-8; -*- - -from sqlalchemy import orm -import sqlalchemy_continuum as continuum - -from wuttjamaican.util import make_true_uuid -from wutta_continuum.testing import VersionTestCase - -from wuttasync.importing import versions as mod, Importer - - -class TestFromWuttaToVersions(VersionTestCase): - - def make_handler(self, **kwargs): - return mod.FromWuttaToVersions(self.config, **kwargs) - - def test_consume_kwargs(self): - - # no comment by default - handler = self.make_handler() - kw = handler.consume_kwargs({}) - self.assertEqual(kw, {}) - self.assertIsNone(handler.continuum_comment) - - # but can provide one - handler = self.make_handler() - kw = handler.consume_kwargs({"comment": "yeehaw"}) - self.assertEqual(kw, {}) - self.assertEqual(handler.continuum_comment, "yeehaw") - - def test_begin_target_transaction(self): - model = self.app.model - txncls = continuum.transaction_class(model.User) - - # basic / defaults - handler = self.make_handler() - self.assertIsNone(handler.continuum_uow) - self.assertIsNone(handler.continuum_txn) - handler.begin_target_transaction() - self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork) - self.assertIsInstance(handler.continuum_txn, txncls) - # nb. no comment - self.assertIsNone(handler.continuum_txn.meta.get("comment")) - - # with comment - handler = self.make_handler() - handler.continuum_comment = "yeehaw" - handler.begin_target_transaction() - self.assertIn("comment", handler.continuum_txn.meta) - self.assertEqual(handler.continuum_txn.meta["comment"], "yeehaw") - - def test_get_importer_kwargs(self): - handler = self.make_handler() - handler.begin_target_transaction() - - kw = handler.get_importer_kwargs("User") - self.assertIn("continuum_txn", kw) - self.assertIs(kw["continuum_txn"], handler.continuum_txn) - - def test_make_importer_factory(self): - model = self.app.model - handler = self.make_handler() - - # versioned class - factory = handler.make_importer_factory(model.User, "User") - self.assertTrue(issubclass(factory, mod.FromWuttaToVersionBase)) - self.assertIs(factory.source_model_class, model.User) - self.assertIs(factory.model_class, continuum.version_class(model.User)) - - # non-versioned - factory = handler.make_importer_factory(model.Upgrade, "Upgrade") - self.assertIsNone(factory) - - def test_define_importers(self): - handler = self.make_handler() - - importers = handler.define_importers() - self.assertIn("User", importers) - self.assertIn("Person", importers) - self.assertNotIn("Upgrade", importers) - - -class TestFromWuttaToVersionBase(VersionTestCase): - - def make_importer(self, model_class=None, **kwargs): - imp = mod.FromWuttaToVersionBase(self.config, **kwargs) - if model_class: - imp.model_class = model_class - return imp - - def test_get_simple_fields(self): - model = self.app.model - vercls = continuum.version_class(model.User) - - # first confirm what a "normal" importer would do - imp = Importer(self.config, model_class=vercls) - fields = imp.get_simple_fields() - self.assertIn("username", fields) - self.assertIn("person_uuid", fields) - self.assertIn("transaction_id", fields) - self.assertIn("operation_type", fields) - self.assertIn("end_transaction_id", fields) - - # now test what the "version" importer does - imp = self.make_importer(model_class=vercls) - fields = imp.get_simple_fields() - self.assertIn("username", fields) - self.assertIn("person_uuid", fields) - self.assertNotIn("transaction_id", fields) - self.assertNotIn("operation_type", fields) - self.assertNotIn("end_transaction_id", fields) - - def test_get_target_query(self): - model = self.app.model - vercls = continuum.version_class(model.User) - imp = self.make_importer(model_class=vercls, target_session=self.session) - - # TODO: not sure what else to test here.. - query = imp.get_target_query() - self.assertIsInstance(query, orm.Query) - - def test_normalize_target_object(self): - model = self.app.model - vercls = continuum.version_class(model.User) - imp = self.make_importer(model_class=vercls) - - user = model.User(username="fred") - self.session.add(user) - self.session.commit() - version = user.versions[0] - - # version object should be embedded in data dict - data = imp.normalize_target_object(version) - self.assertIsInstance(data, dict) - self.assertIn("_objref", data) - self.assertIs(data["_objref"], version) - - # but normal object is not embedded - data = imp.normalize_target_object(user) - self.assertIsInstance(data, dict) - self.assertNotIn("_version", data) - - def test_make_version(self): - model = self.app.model - vercls = continuum.version_class(model.User) - - user = model.User(username="fred") - self.session.add(user) - self.session.commit() - - handler = mod.FromWuttaToVersions(self.config) - handler.begin_target_transaction() - handler.target_session.close() - handler.target_session = self.session - - imp = self.make_importer( - model_class=vercls, - fields=["uuid", "username"], - keys=("uuid",), - target_session=self.session, - continuum_txn=handler.continuum_txn, - ) - - data = {"uuid": user.uuid, "username": "freddie"} - version = imp.make_version(data, continuum.Operation.UPDATE) - self.assertIsInstance(version, vercls) - self.assertEqual(version.uuid, user.uuid) - self.assertEqual(version.username, "freddie") - self.assertIn(version, self.session) - self.assertIs(version.transaction, imp.continuum_txn) - self.assertEqual(version.operation_type, continuum.Operation.UPDATE) - - def test_create_target_object(self): - model = self.app.model - vercls = continuum.version_class(model.User) - - handler = mod.FromWuttaToVersions(self.config) - handler.begin_target_transaction() - handler.target_session.close() - handler.target_session = self.session - - imp = self.make_importer( - model_class=vercls, - fields=["uuid", "username"], - keys=("uuid",), - target_session=self.session, - continuum_txn=handler.continuum_txn, - ) - - source_data = {"uuid": make_true_uuid(), "username": "bettie"} - self.assertEqual(self.session.query(vercls).count(), 0) - version = imp.create_target_object((source_data["uuid"], 1), source_data) - self.assertEqual(self.session.query(vercls).count(), 1) - self.assertEqual(version.transaction_id, imp.continuum_txn.id) - self.assertEqual(version.operation_type, continuum.Operation.INSERT) - self.assertIsNone(version.end_transaction_id) - - def test_update_target_object(self): - model = self.app.model - vercls = continuum.version_class(model.User) - - user = model.User(username="fred") - self.session.add(user) - self.session.commit() - version1 = user.versions[0] - - handler = mod.FromWuttaToVersions(self.config) - handler.begin_target_transaction() - handler.target_session.close() - handler.target_session = self.session - - imp = self.make_importer( - model_class=vercls, - fields=["uuid", "username"], - keys=("uuid",), - target_session=self.session, - continuum_txn=handler.continuum_txn, - ) - - source_data = {"uuid": user.uuid, "username": "freddie"} - target_data = imp.normalize_target_object(version1) - self.assertEqual(self.session.query(vercls).count(), 1) - self.assertIsNone(version1.end_transaction_id) - version2 = imp.update_target_object( - version1, source_data, target_data=target_data - ) - self.assertEqual(self.session.query(vercls).count(), 2) - self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id) - self.assertEqual(version2.transaction_id, imp.continuum_txn.id) - self.assertEqual(version2.operation_type, continuum.Operation.UPDATE) - self.assertIsNone(version2.end_transaction_id) - - def test_delete_target_object(self): - model = self.app.model - vercls = continuum.version_class(model.User) - - user = model.User(username="fred") - self.session.add(user) - self.session.commit() - version1 = user.versions[0] - - handler = mod.FromWuttaToVersions(self.config) - handler.begin_target_transaction() - handler.target_session.close() - handler.target_session = self.session - - imp = self.make_importer( - model_class=vercls, - fields=["uuid", "username"], - keys=("uuid",), - target_session=self.session, - continuum_txn=handler.continuum_txn, - ) - - self.assertEqual(self.session.query(vercls).count(), 1) - self.assertIsNone(version1.end_transaction_id) - version2 = imp.delete_target_object(version1) - self.assertEqual(self.session.query(vercls).count(), 2) - self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id) - self.assertEqual(version2.transaction_id, imp.continuum_txn.id) - self.assertEqual(version2.operation_type, continuum.Operation.DELETE) - self.assertIsNone(version2.end_transaction_id) diff --git a/tests/importing/test_wutta.py b/tests/importing/test_wutta.py index 1533605..4d6fdd2 100644 --- a/tests/importing/test_wutta.py +++ b/tests/importing/test_wutta.py @@ -1,3 +1,38 @@ # -*- coding: utf-8; -*- +from unittest.mock import patch + +from wuttjamaican.testing import DataTestCase + from wuttasync.importing import wutta as mod + + +class TestToWuttaHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.ToWuttaHandler(self.config, **kwargs) + + def test_get_target_title(self): + handler = self.make_handler() + + # uses app title by default + self.config.setdefault("wutta.app_title", "What About This") + self.assertEqual(handler.get_target_title(), "What About This") + + # or generic default if present + handler.generic_target_title = "WHATABOUTTHIS" + self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS") + + # but prefer specific title if present + handler.target_title = "what_about_this" + self.assertEqual(handler.get_target_title(), "what_about_this") + + def test_make_target_session(self): + handler = self.make_handler() + + # makes "new" (mocked in our case) app session + with patch.object(self.app, "make_session") as make_session: + make_session.return_value = self.session + session = handler.make_target_session() + make_session.assert_called_once_with() + self.assertIs(session, self.session) diff --git a/tests/test_app.py b/tests/test_app.py deleted file mode 100644 index ea2b5e6..0000000 --- a/tests/test_app.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8; -*- - -from wuttjamaican.testing import ConfigTestCase - -from wuttasync import app as mod -from wuttasync.importing import ImportHandler -from wuttasync.importing.csv import FromCsvToWutta - - -class FromFooToBar(ImportHandler): - source_key = "foo" - target_key = "bar" - - -class FromCsvToPoser(FromCsvToWutta): - pass - - -class TestWuttaSyncAppProvider(ConfigTestCase): - - def test_get_all_import_handlers(self): - - # by default our custom handler is not found - handlers = self.app.get_all_import_handlers() - self.assertIn(FromCsvToWutta, handlers) - self.assertNotIn(FromFooToBar, handlers) - - # make sure if we configure a custom handler, it is found - self.config.setdefault( - "wuttasync.importing.import.to_wutta.from_csv.handler", - "tests.test_app:FromFooToBar", - ) - handlers = self.app.get_all_import_handlers() - self.assertIn(FromCsvToWutta, handlers) - self.assertIn(FromFooToBar, handlers) - - def test_get_designated_import_handler_spec(self): - - # fetch of unknown key returns none - spec = self.app.get_designated_import_handler_spec("test01") - self.assertIsNone(spec) - - # unless we require it, in which case, error - self.assertRaises( - ValueError, - self.app.get_designated_import_handler_spec, - "test01", - require=True, - ) - - # we configure one for whatever key we like - self.config.setdefault( - "wuttasync.importing.test02.handler", "tests.test_app:FromBarToFoo" - ) - spec = self.app.get_designated_import_handler_spec("test02") - self.assertEqual(spec, "tests.test_app:FromBarToFoo") - - # we can also define a "default" designated handler - self.config.setdefault( - "wuttasync.importing.test03.default_handler", - "tests.test_app:FromBarToFoo", - ) - spec = self.app.get_designated_import_handler_spec("test03") - self.assertEqual(spec, "tests.test_app:FromBarToFoo") - - def test_get_designated_import_handlers(self): - - # some designated handlers exist, but not our custom handler - handlers = self.app.get_designated_import_handlers() - csv_handlers = [ - h for h in handlers if h.get_key() == "import.to_wutta.from_csv" - ] - self.assertEqual(len(csv_handlers), 1) - csv_handler = csv_handlers[0] - self.assertIsInstance(csv_handler, FromCsvToWutta) - self.assertFalse(isinstance(csv_handler, FromCsvToPoser)) - self.assertFalse( - any([h.get_key() == "import.to_bar.from_foo" for h in handlers]) - ) - self.assertFalse(any([isinstance(h, FromFooToBar) for h in handlers])) - self.assertFalse(any([isinstance(h, FromCsvToPoser) for h in handlers])) - self.assertTrue( - any([h.get_key() == "import.to_versions.from_wutta" for h in handlers]) - ) - - # but we can make custom designated - self.config.setdefault( - "wuttasync.importing.import.to_wutta.from_csv.handler", - "tests.test_app:FromCsvToPoser", - ) - handlers = self.app.get_designated_import_handlers() - csv_handlers = [ - h for h in handlers if h.get_key() == "import.to_wutta.from_csv" - ] - self.assertEqual(len(csv_handlers), 1) - csv_handler = csv_handlers[0] - self.assertIsInstance(csv_handler, FromCsvToWutta) - self.assertIsInstance(csv_handler, FromCsvToPoser) - self.assertTrue( - any([h.get_key() == "import.to_versions.from_wutta" for h in handlers]) - ) - - def test_get_import_handler(self): - - # make sure a basic fetch works - handler = self.app.get_import_handler("import.to_wutta.from_csv") - self.assertIsInstance(handler, FromCsvToWutta) - self.assertFalse(isinstance(handler, FromCsvToPoser)) - - # and make sure custom override works - self.config.setdefault( - "wuttasync.importing.import.to_wutta.from_csv.handler", - "tests.test_app:FromCsvToPoser", - ) - handler = self.app.get_import_handler("import.to_wutta.from_csv") - self.assertIsInstance(handler, FromCsvToWutta) - self.assertIsInstance(handler, FromCsvToPoser) - - # unknown importer cannot be found - handler = self.app.get_import_handler("bogus") - self.assertIsNone(handler) - - # and if we require it, error will raise - self.assertRaises( - ValueError, self.app.get_import_handler, "bogus", require=True - ) diff --git a/tests/test_emails.py b/tests/test_emails.py deleted file mode 100644 index 9494753..0000000 --- a/tests/test_emails.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8; -*- - -from wuttjamaican.testing import ConfigTestCase - -from wuttasync import emails as mod -from wuttasync.importing import ImportHandler -from wuttasync.testing import ImportExportWarningTestCase - - -class FromFooToWutta(ImportHandler): - pass - - -class TestImportExportWarning(ConfigTestCase): - - def make_setting(self, factory=None): - if not factory: - factory = mod.ImportExportWarning - setting = factory(self.config) - return setting - - def test_get_description(self): - self.config.setdefault("wutta.app_title", "Wutta Poser") - setting = self.make_setting() - setting.import_handler_key = "import.to_wutta.from_csv" - self.assertEqual( - setting.get_description(), - "Diff warning email for importing CSV → Wutta Poser", - ) - - def test_get_default_subject(self): - self.config.setdefault("wutta.app_title", "Wutta Poser") - setting = self.make_setting() - setting.import_handler_key = "import.to_wutta.from_csv" - self.assertEqual(setting.get_default_subject(), "Changes for CSV → Wutta Poser") - - def test_get_import_handler(self): - - # nb. typical name pattern - class import_to_wutta_from_foo_warning(mod.ImportExportWarning): - pass - - # nb. name does not match spec pattern - class import_to_wutta_from_bar_blah(mod.ImportExportWarning): - pass - - # register our import handler - self.config.setdefault( - "wuttasync.importing.import.to_wutta.from_foo.handler", - "tests.test_emails:FromFooToWutta", - ) - - # error if spec/key not discoverable - setting = self.make_setting(import_to_wutta_from_bar_blah) - self.assertRaises(ValueError, setting.get_import_handler) - - # can lookup by name (auto-spec) - setting = self.make_setting(import_to_wutta_from_foo_warning) - handler = setting.get_import_handler() - self.assertIsInstance(handler, FromFooToWutta) - - # can lookup by explicit spec - setting = self.make_setting(import_to_wutta_from_bar_blah) - setting.import_handler_spec = "tests.test_emails:FromFooToWutta" - handler = setting.get_import_handler() - self.assertIsInstance(handler, FromFooToWutta) - - # can lookup by explicit key - setting = self.make_setting(import_to_wutta_from_bar_blah) - setting.import_handler_key = "import.to_wutta.from_foo" - handler = setting.get_import_handler() - self.assertIsInstance(handler, FromFooToWutta) - - -class TestEmailSettings(ImportExportWarningTestCase): - - def test_import_to_versions_from_wutta_warning(self): - self.do_test_preview("import_to_versions_from_wutta_warning") - - def test_import_to_wutta_from_csv_warning(self): - self.do_test_preview("import_to_wutta_from_csv_warning")