Compare commits
No commits in common. "8c5918b9fb9572b096c60e989aa562b361f8c3b5" and "c38cd2c1792d2a90adbdb6112c81afe2cf1f8bff" have entirely different histories.
8c5918b9fb
...
c38cd2c179
33 changed files with 116 additions and 2588 deletions
19
CHANGELOG.md
19
CHANGELOG.md
|
|
@ -5,25 +5,6 @@ All notable changes to WuttaSync will be documented in this file.
|
|||
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## v0.3.0 (2025-12-20)
|
||||
|
||||
### Feat
|
||||
|
||||
- add `warnings` mode for import/export handlers, commands
|
||||
- add the `import-versions` command, handler logic
|
||||
|
||||
### Fix
|
||||
|
||||
- run all models when none specified, for import/export commands
|
||||
- allow passing just `key` to ImportCommandHandler
|
||||
- add `--comment` param for `import-versions` command
|
||||
- add basic data type coercion for CSV -> SQLAlchemy import
|
||||
- refactor some more for tests + pylint
|
||||
- refactor per pylint; add to tox
|
||||
- format all code with black
|
||||
- tweak logging when deleting object
|
||||
- add logging when deleting target object
|
||||
|
||||
## v0.2.1 (2025-06-29)
|
||||
|
||||
### Fix
|
||||
|
|
|
|||
|
|
@ -1,6 +0,0 @@
|
|||
|
||||
``wuttasync.app``
|
||||
=================
|
||||
|
||||
.. automodule:: wuttasync.app
|
||||
:members:
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
|
||||
``wuttasync.cli.import_versions``
|
||||
=================================
|
||||
|
||||
.. automodule:: wuttasync.cli.import_versions
|
||||
:members:
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
|
||||
``wuttasync.emails``
|
||||
====================
|
||||
|
||||
.. automodule:: wuttasync.emails
|
||||
:members:
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
|
||||
``wuttasync.importing.versions``
|
||||
================================
|
||||
|
||||
.. automodule:: wuttasync.importing.versions
|
||||
:members:
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
|
||||
``wuttasync.testing``
|
||||
=====================
|
||||
|
||||
.. automodule:: wuttasync.testing
|
||||
:members:
|
||||
|
|
@ -31,13 +31,6 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
|||
intersphinx_mapping = {
|
||||
"python": ("https://docs.python.org/3/", None),
|
||||
"rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None),
|
||||
"sqlalchemy": ("http://docs.sqlalchemy.org/en/latest/", None),
|
||||
"sqlalchemy-continuum": (
|
||||
"https://sqlalchemy-continuum.readthedocs.io/en/latest/",
|
||||
None,
|
||||
),
|
||||
"sqlalchemy-utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None),
|
||||
"wutta-continuum": ("https://docs.wuttaproject.org/wutta-continuum/", None),
|
||||
"wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,26 +6,6 @@ Glossary
|
|||
.. glossary::
|
||||
:sorted:
|
||||
|
||||
import/export key
|
||||
Unique key representing a particular type of import/export job,
|
||||
i.e. the source/target and orientation (import vs. export).
|
||||
|
||||
For instance "Wutta → CSV export" uses the key:
|
||||
``export.to_csv.from_wutta``
|
||||
|
||||
More than one :term:`import handler` can share a key, e.g. one
|
||||
may subclass another and inherit the key.
|
||||
|
||||
However only one handler is "designated" for a given key; it will
|
||||
be used by default for running those jobs.
|
||||
|
||||
This key is used for lookup in
|
||||
:meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`.
|
||||
|
||||
See also
|
||||
:meth:`~wuttasync.importing.handlers.ImportHandler.get_key()`
|
||||
method on the import/export handler.
|
||||
|
||||
import handler
|
||||
This a type of :term:`handler` which is responsible for a
|
||||
particular set of data import/export task(s).
|
||||
|
|
|
|||
|
|
@ -67,21 +67,16 @@ cf. :doc:`rattail-manual:data/sync/index`.
|
|||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:caption: Package API
|
||||
:caption: API
|
||||
|
||||
api/wuttasync
|
||||
api/wuttasync.app
|
||||
api/wuttasync.cli
|
||||
api/wuttasync.cli.base
|
||||
api/wuttasync.cli.import_csv
|
||||
api/wuttasync.cli.import_versions
|
||||
api/wuttasync.emails
|
||||
api/wuttasync.importing
|
||||
api/wuttasync.importing.base
|
||||
api/wuttasync.importing.csv
|
||||
api/wuttasync.importing.handlers
|
||||
api/wuttasync.importing.model
|
||||
api/wuttasync.importing.versions
|
||||
api/wuttasync.importing.wutta
|
||||
api/wuttasync.testing
|
||||
api/wuttasync.util
|
||||
|
|
|
|||
|
|
@ -25,24 +25,3 @@ types may not behave as expected etc.
|
|||
Defined in: :mod:`wuttasync.cli.import_csv`
|
||||
|
||||
.. program-output:: wutta import-csv --help
|
||||
|
||||
|
||||
.. _wutta-import-versions:
|
||||
|
||||
``wutta import-versions``
|
||||
-------------------------
|
||||
|
||||
Import latest data to version tables, for the Wutta :term:`app
|
||||
database`.
|
||||
|
||||
The purpose of this is to ensure version tables accurately reflect
|
||||
the current "live" data set, for given table(s). It is only
|
||||
relevant/usable if versioning is configured and enabled. For more
|
||||
on that see :doc:`wutta-continuum:index`.
|
||||
|
||||
This command can check/update version tables for any versioned class
|
||||
in the :term:`app model`.
|
||||
|
||||
Defined in: :mod:`wuttasync.cli.import_versions`
|
||||
|
||||
.. program-output:: wutta import-versions --help
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "WuttaSync"
|
||||
version = "0.3.0"
|
||||
version = "0.2.1"
|
||||
description = "Wutta Framework for data import/export and real-time sync"
|
||||
readme = "README.md"
|
||||
authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]
|
||||
|
|
@ -26,25 +26,17 @@ classifiers = [
|
|||
]
|
||||
requires-python = ">= 3.8"
|
||||
dependencies = [
|
||||
"humanize",
|
||||
"makefun",
|
||||
"SQLAlchemy-Utils",
|
||||
"WuttJamaican[db]>=0.27.0",
|
||||
"WuttJamaican[db]>=0.16.2",
|
||||
]
|
||||
|
||||
|
||||
[project.optional-dependencies]
|
||||
docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
|
||||
tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"]
|
||||
tests = ["pylint", "pytest", "pytest-cov", "tox"]
|
||||
|
||||
|
||||
[project.entry-points."wutta.app.providers"]
|
||||
wuttasync = "wuttasync.app:WuttaSyncAppProvider"
|
||||
|
||||
[project.entry-points."wuttasync.importing"]
|
||||
"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions"
|
||||
"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta"
|
||||
|
||||
[project.entry-points."wutta.typer_imports"]
|
||||
wuttasync = "wuttasync.cli"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,225 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
App handler supplement for WuttaSync
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from wuttjamaican.app import AppProvider
|
||||
from wuttjamaican.util import load_entry_points
|
||||
|
||||
|
||||
class WuttaSyncAppProvider(AppProvider):
|
||||
"""
|
||||
The :term:`app provider` for WuttaSync.
|
||||
|
||||
This adds some methods to the :term:`app handler`, which are
|
||||
specific to import/export.
|
||||
|
||||
It also declares some :term:`email modules <email module>` and
|
||||
:term:`email templates <email template>` for the app.
|
||||
|
||||
We have two concerns when doing lookups etc. for import/export
|
||||
handlers:
|
||||
|
||||
* which handlers are *available* - i.e. they exist and are
|
||||
discoverable
|
||||
* which handlers are *designated* - only one designated handler
|
||||
per key
|
||||
|
||||
All "available" handlers will have a key, but some keys may be
|
||||
referenced by multiple handlers. For each key, only one handler
|
||||
can be "designated" - there is a default, but config can override.
|
||||
"""
|
||||
|
||||
email_modules = ["wuttasync.emails"]
|
||||
email_templates = ["wuttasync:email-templates"]
|
||||
|
||||
def get_all_import_handlers(self):
|
||||
"""
|
||||
Returns *all* :term:`import/export handler <import handler>`
|
||||
*classes* which are known to exist, i.e. are discoverable.
|
||||
|
||||
See also :meth:`get_import_handler()` and
|
||||
:meth:`get_designated_import_handlers()`.
|
||||
|
||||
The discovery process is as follows:
|
||||
|
||||
* load handlers from registered entry points
|
||||
* check config for designated handlers
|
||||
|
||||
Checking for designated handler config is not a reliable way
|
||||
to discover handlers, but it's done just in case any new ones
|
||||
might be found.
|
||||
|
||||
Registration via entry points is the only way to ensure a
|
||||
handler is discoverable. The entry point group name is always
|
||||
``wuttasync.importing`` regardless of :term:`app name`;
|
||||
entries are like ``"handler_key" = "handler_spec"``. For
|
||||
example:
|
||||
|
||||
.. code-block:: toml
|
||||
|
||||
[project.entry-points."wuttasync.importing"]
|
||||
"export.to_csv.from_poser" = "poser.exporting.csv:FromPoserToCsv"
|
||||
"import.to_poser.from_csv" = "poser.importing.csv:FromCsvToPoser"
|
||||
|
||||
:returns: List of all import/export handler classes
|
||||
"""
|
||||
# first load all "registered" Handler classes
|
||||
factories = load_entry_points("wuttasync.importing", ignore_errors=True)
|
||||
|
||||
# organize registered classes by spec
|
||||
specs = {factory.get_spec(): factory for factory in factories.values()}
|
||||
|
||||
# many handlers may not be registered per se, but may be
|
||||
# designated via config. so try to include those too
|
||||
for factory in factories.values():
|
||||
spec = self.get_designated_import_handler_spec(factory.get_key())
|
||||
if spec and spec not in specs:
|
||||
specs[spec] = self.app.load_object(spec)
|
||||
|
||||
# flatten back to simple list of classes
|
||||
factories = list(specs.values())
|
||||
return factories
|
||||
|
||||
def get_designated_import_handler_spec(self, key, require=False):
|
||||
"""
|
||||
Returns the designated import/export handler :term:`spec`
|
||||
string for the given type key.
|
||||
|
||||
This just checks config for the designated handler, using the
|
||||
``wuttasync.importing`` prefix regardless of :term:`app name`.
|
||||
For instance:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[wuttasync.importing]
|
||||
export.to_csv.from_poser.handler = poser.exporting.csv:FromPoserToCsv
|
||||
import.to_poser.from_csv.handler = poser.importing.csv:FromCsvToPoser
|
||||
|
||||
See also :meth:`get_designated_import_handlers()` and
|
||||
:meth:`get_import_handler()`.
|
||||
|
||||
:param key: Unique key indicating the type of import/export
|
||||
handler.
|
||||
|
||||
:param require: Flag indicating whether an error should be raised if no
|
||||
handler is found.
|
||||
|
||||
:returns: Spec string for the designated handler. If none is
|
||||
configured, then ``None`` is returned *unless* the
|
||||
``require`` param is true, in which case an error is
|
||||
raised.
|
||||
"""
|
||||
spec = self.config.get(f"wuttasync.importing.{key}.handler")
|
||||
if spec:
|
||||
return spec
|
||||
|
||||
spec = self.config.get(f"wuttasync.importing.{key}.default_handler")
|
||||
if spec:
|
||||
return spec
|
||||
|
||||
if require:
|
||||
raise ValueError(f"Cannot locate import handler spec for key: {key}")
|
||||
return None
|
||||
|
||||
def get_designated_import_handlers(self):
|
||||
"""
|
||||
Returns all *designated* import/export handler *instances*.
|
||||
|
||||
Each import/export handler has a "key" which indicates the
|
||||
"type" of import/export job it performs. For instance the CSV
|
||||
→ Wutta import has the key: ``import.to_wutta.from_csv``
|
||||
|
||||
More than one handler can be defined for that key; however
|
||||
only one such handler will be "designated" for each key.
|
||||
|
||||
This method first loads *all* available import handlers, then
|
||||
organizes them by key, and tries to determine which handler
|
||||
should be designated for each key.
|
||||
|
||||
See also :meth:`get_all_import_handlers()` and
|
||||
:meth:`get_designated_import_handler_spec()`.
|
||||
|
||||
:returns: List of designated import/export handler instances
|
||||
"""
|
||||
grouped = OrderedDict()
|
||||
for factory in self.get_all_import_handlers():
|
||||
key = factory.get_key()
|
||||
grouped.setdefault(key, []).append(factory)
|
||||
|
||||
def find_designated(key, group):
|
||||
spec = self.get_designated_import_handler_spec(key)
|
||||
if spec:
|
||||
for factory in group:
|
||||
if factory.get_spec() == spec:
|
||||
return factory
|
||||
if len(group) == 1:
|
||||
return group[0]
|
||||
return None
|
||||
|
||||
designated = []
|
||||
for key, group in grouped.items():
|
||||
factory = find_designated(key, group)
|
||||
if factory:
|
||||
handler = factory(self.config)
|
||||
designated.append(handler)
|
||||
|
||||
return designated
|
||||
|
||||
def get_import_handler(self, key, require=False, **kwargs):
|
||||
"""
|
||||
Returns the designated :term:`import/export handler <import
|
||||
handler>` instance for the given :term:`import/export key`.
|
||||
|
||||
See also :meth:`get_all_import_handlers()` and
|
||||
:meth:`get_designated_import_handlers()`.
|
||||
|
||||
:param key: Key indicating the type of import/export handler,
|
||||
e.g. ``"import.to_wutta.from_csv"``
|
||||
|
||||
:param require: Set this to true if you want an error raised
|
||||
when no handler is found.
|
||||
|
||||
:returns: The import/export handler instance. If no handler
|
||||
is found, then ``None`` is returned, unless ``require``
|
||||
param is true, in which case error is raised.
|
||||
"""
|
||||
# first try to fetch the handler per designated spec
|
||||
spec = self.get_designated_import_handler_spec(key, **kwargs)
|
||||
if spec:
|
||||
factory = self.app.load_object(spec)
|
||||
return factory(self.config)
|
||||
|
||||
# nothing was designated, so leverage logic which already
|
||||
# sorts out which handler is "designated" for given key
|
||||
designated = self.get_designated_import_handlers()
|
||||
for handler in designated:
|
||||
if handler.get_key() == key:
|
||||
return handler
|
||||
|
||||
if require:
|
||||
raise ValueError(f"Cannot locate import handler for key: {key}")
|
||||
return None
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
# Copyright © 2024 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
|
|
@ -34,4 +34,3 @@ from .base import import_command, file_import_command, ImportCommandHandler
|
|||
|
||||
# nb. must bring in all modules for discovery to work
|
||||
from . import import_csv
|
||||
from . import import_versions
|
||||
|
|
|
|||
|
|
@ -50,35 +50,19 @@ class ImportCommandHandler(GenericHandler):
|
|||
create this handler and call its :meth:`run()` method.
|
||||
|
||||
This handler does not know how to import/export data, but it knows
|
||||
how to make its :attr:`import_handler` do it. Likewise, the
|
||||
import handler is not "CLI-aware" - so this provides the glue.
|
||||
how to make its :attr:`import_handler` do it.
|
||||
|
||||
:param import_handler: During construction, caller can specify the
|
||||
:attr:`import_handler` as any of:
|
||||
|
||||
* import handler instance
|
||||
* import handler factory (e.g. class)
|
||||
* import handler :term:`spec`
|
||||
* import handler spec (cf. :func:`~wuttjamaican:wuttjamaican.util.load_object()`)
|
||||
|
||||
:param key: Optional :term:`import/export key` to use for handler
|
||||
lookup. Only used if ``import_handler`` param is not set.
|
||||
|
||||
Typical usage for custom commands will be to provide the spec::
|
||||
For example::
|
||||
|
||||
handler = ImportCommandHandler(
|
||||
config, "poser.importing.foo:FromFooToPoser"
|
||||
)
|
||||
|
||||
Library authors may prefer to use the import/export key; this lets
|
||||
the command work with any designated handler::
|
||||
|
||||
handler = ImportCommandHandler(
|
||||
config, key="import.to_poser.from_foo"
|
||||
)
|
||||
|
||||
See also
|
||||
:meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`
|
||||
which does the lookup by key.
|
||||
config, import_handler='wuttasync.importing.csv:FromCsvToWutta')
|
||||
"""
|
||||
|
||||
import_handler = None
|
||||
|
|
@ -87,7 +71,7 @@ class ImportCommandHandler(GenericHandler):
|
|||
invoked when command runs. See also :meth:`run()`.
|
||||
"""
|
||||
|
||||
def __init__(self, config, import_handler=None, key=None):
|
||||
def __init__(self, config, import_handler=None):
|
||||
super().__init__(config)
|
||||
|
||||
if import_handler:
|
||||
|
|
@ -99,9 +83,6 @@ class ImportCommandHandler(GenericHandler):
|
|||
factory = self.app.load_object(import_handler)
|
||||
self.import_handler = factory(self.config)
|
||||
|
||||
elif key:
|
||||
self.import_handler = self.app.get_import_handler(key, require=True)
|
||||
|
||||
def run(self, params, progress=None): # pylint: disable=unused-argument
|
||||
"""
|
||||
Run the import/export job(s) based on command line params.
|
||||
|
|
@ -125,15 +106,15 @@ class ImportCommandHandler(GenericHandler):
|
|||
return
|
||||
|
||||
# otherwise process some data
|
||||
log.debug("using handler: %s", self.import_handler.get_spec())
|
||||
kw = dict(params)
|
||||
models = kw.pop("models")
|
||||
if not models:
|
||||
models = list(self.import_handler.importers)
|
||||
log.debug("using handler: %s", self.import_handler.get_spec())
|
||||
# TODO: need to use all/default models if none specified
|
||||
# (and should know models by now for logging purposes)
|
||||
log.debug(
|
||||
"%s %s for models: %s",
|
||||
self.import_handler.actioning,
|
||||
self.import_handler.get_title(),
|
||||
"running %s %s for: %s",
|
||||
self.import_handler,
|
||||
self.import_handler.orientation.value,
|
||||
", ".join(models),
|
||||
)
|
||||
log.debug("params are: %s", kw)
|
||||
|
|
@ -146,16 +127,15 @@ class ImportCommandHandler(GenericHandler):
|
|||
|
||||
This is what happens when command line has ``--list-models``.
|
||||
"""
|
||||
sys.stdout.write("\nALL MODELS:\n")
|
||||
sys.stdout.write("ALL MODELS:\n")
|
||||
sys.stdout.write("==============================\n")
|
||||
for key in self.import_handler.importers:
|
||||
sys.stdout.write(key)
|
||||
sys.stdout.write("\n")
|
||||
sys.stdout.write("==============================\n")
|
||||
sys.stdout.write(f"for {self.import_handler.get_title()}\n\n")
|
||||
|
||||
|
||||
def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments,too-many-locals
|
||||
def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments
|
||||
models: Annotated[
|
||||
Optional[List[str]],
|
||||
typer.Argument(
|
||||
|
|
@ -237,27 +217,6 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
|
|||
help="Max number of *any* target record changes which may occur (per model)."
|
||||
),
|
||||
] = None,
|
||||
warnings: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
"--warn",
|
||||
"-W",
|
||||
help="Expect no changes; warn (email the diff) if any occur.",
|
||||
),
|
||||
] = False,
|
||||
warnings_recipients: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--recip", help="Override the recipient(s) for diff warning email."
|
||||
),
|
||||
] = None,
|
||||
warnings_max_diffs: Annotated[
|
||||
int,
|
||||
typer.Option(
|
||||
"--max-diffs",
|
||||
help="Max number of record diffs to show (per model) in warning email.",
|
||||
),
|
||||
] = 15,
|
||||
dry_run: Annotated[
|
||||
bool,
|
||||
typer.Option(
|
||||
|
|
|
|||
|
|
@ -38,5 +38,7 @@ def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
|
|||
Import data from CSV file(s) to Wutta DB
|
||||
"""
|
||||
config = ctx.parent.wutta_config
|
||||
handler = ImportCommandHandler(config, key="import.to_wutta.from_csv")
|
||||
handler = ImportCommandHandler(
|
||||
config, import_handler="wuttasync.importing.csv:FromCsvToWutta"
|
||||
)
|
||||
handler.run(ctx.params)
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
See also: :ref:`wutta-import-versions`
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import rich
|
||||
import typer
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from wuttjamaican.cli import wutta_typer
|
||||
|
||||
from .base import import_command, ImportCommandHandler
|
||||
|
||||
|
||||
@wutta_typer.command()
|
||||
@import_command
|
||||
def import_versions( # pylint: disable=unused-argument
|
||||
ctx: typer.Context,
|
||||
comment: Annotated[
|
||||
str,
|
||||
typer.Option("--comment", "-m", help="Comment to set on the transaction."),
|
||||
] = "import catch-up versions",
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Import latest data to version tables, for Wutta DB
|
||||
"""
|
||||
config = ctx.parent.wutta_config
|
||||
app = config.get_app()
|
||||
|
||||
# warn/exit if libs are not installed
|
||||
try:
|
||||
import wutta_continuum # pylint: disable=import-outside-toplevel,unused-import
|
||||
except ImportError: # pragma: no cover
|
||||
rich.print(
|
||||
"\n\t[bold yellow]Wutta-Continum is not installed![/bold yellow]\n"
|
||||
"\n\tIf you want it, run: pip install Wutta-Continuum\n"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# warn/exit if feature disabled
|
||||
if not app.continuum_is_enabled(): # pragma: no cover
|
||||
rich.print(
|
||||
"\n\t[bold yellow]Wutta-Continum is not enabled![/bold yellow]\n"
|
||||
"\n\tIf you want it, see: https://docs.wuttaproject.org/wutta-continuum/\n"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
handler = ImportCommandHandler(config, key="import.to_versions.from_wutta")
|
||||
handler.run(ctx.params)
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
## -*- coding: utf-8; -*-
|
||||
<html>
|
||||
<body>
|
||||
<h3>Diff warning for ${title} (${handler.actioning})</h3>
|
||||
|
||||
<p style="font-style: italic;">
|
||||
% if dry_run:
|
||||
<span style="font-weight: bold;">DRY RUN</span>
|
||||
- these changes have not yet happened
|
||||
% else:
|
||||
<span style="font-weight: bold;">LIVE RUN</span>
|
||||
- these changes already happened
|
||||
% endif
|
||||
</p>
|
||||
|
||||
<ul>
|
||||
% for model, (created, updated, deleted) in changes.items():
|
||||
<li>
|
||||
<a href="#${model}">${model}</a> -
|
||||
${app.render_quantity(len(created))} created;
|
||||
${app.render_quantity(len(updated))} updated;
|
||||
${app.render_quantity(len(deleted))} deleted
|
||||
</li>
|
||||
% endfor
|
||||
</ul>
|
||||
|
||||
<p>
|
||||
<span style="font-weight: bold;">COMMAND:</span>
|
||||
|
||||
<code>${argv}</code>
|
||||
</p>
|
||||
|
||||
<p>
|
||||
<span style="font-weight: bold;">RUNTIME:</span>
|
||||
|
||||
${runtime} (${runtime_display})
|
||||
</p>
|
||||
|
||||
% for model, (created, updated, deleted) in changes.items():
|
||||
|
||||
<br />
|
||||
<h4>
|
||||
<a name="${model}">${model}</a> -
|
||||
${app.render_quantity(len(created))} created;
|
||||
${app.render_quantity(len(updated))} updated;
|
||||
${app.render_quantity(len(deleted))} deleted
|
||||
</h4>
|
||||
|
||||
<div style="padding-left: 2rem;">
|
||||
|
||||
% for obj, source_data in created[:max_diffs]:
|
||||
<h5>${model} <em>created</em> in ${target_title}: ${obj}</h5>
|
||||
<% diff = make_diff({}, source_data, nature="create") %>
|
||||
<div style="padding-left: 2rem;">
|
||||
${diff.render_html()}
|
||||
</div>
|
||||
% endfor
|
||||
% if len(created) > max_diffs:
|
||||
<h5>${model} - ${app.render_quantity(len(created) - max_diffs)} more records <em>created</em> in ${target_title} - not shown here</h5>
|
||||
% endif
|
||||
|
||||
% for obj, source_data, target_data in updated[:max_diffs]:
|
||||
<h5>${model} <em>updated</em> in ${target_title}: ${obj}</h5>
|
||||
<% diff = make_diff(target_data, source_data, nature="update") %>
|
||||
<div style="padding-left: 2rem;">
|
||||
${diff.render_html()}
|
||||
</div>
|
||||
% endfor
|
||||
% if len(updated) > max_diffs:
|
||||
<h5>${model} - ${app.render_quantity(len(updated) - max_diffs)} more records <em>updated</em> in ${target_title} - not shown here</h5>
|
||||
% endif
|
||||
|
||||
% for obj, target_data in deleted[:max_diffs]:
|
||||
<h5>${model} <em>deleted</em> in ${target_title}: ${obj}</h5>
|
||||
<% diff = make_diff(target_data, {}, nature="delete") %>
|
||||
<div style="padding-left: 2rem;">
|
||||
${diff.render_html()}
|
||||
</div>
|
||||
% endfor
|
||||
% if len(deleted) > max_diffs:
|
||||
<h5>${model} - ${app.render_quantity(len(deleted) - max_diffs)} more records <em>deleted</em> in ${target_title} - not shown here</h5>
|
||||
% endif
|
||||
|
||||
</div>
|
||||
|
||||
% endfor
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,166 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
:term:`Email Settings <email setting>` for WuttaSync
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
from wuttjamaican.email import EmailSetting
|
||||
from wuttjamaican.diffs import Diff
|
||||
|
||||
|
||||
class ImportExportWarning(EmailSetting):
|
||||
"""
|
||||
Base class for import/export diff warnings; sent when unexpected
|
||||
changes occur.
|
||||
|
||||
This inherits from :class:`~wuttjamaican.email.EmailSetting`.
|
||||
"""
|
||||
|
||||
fallback_key = "import_export_warning"
|
||||
"" # suppress docs
|
||||
|
||||
import_handler_spec = None
|
||||
import_handler_key = None
|
||||
|
||||
def get_description(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
handler = self.get_import_handler()
|
||||
return f"Diff warning email for {handler.actioning} {handler.get_title()}"
|
||||
|
||||
def get_default_subject(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
handler = self.get_import_handler()
|
||||
return f"Changes for {handler.get_title()}"
|
||||
|
||||
def get_import_handler(self): # pylint: disable=missing-function-docstring
|
||||
|
||||
# prefer explicit spec, if set
|
||||
if self.import_handler_spec:
|
||||
return self.app.load_object(self.import_handler_spec)(self.config)
|
||||
|
||||
# next try spec lookup, if key set
|
||||
if self.import_handler_key:
|
||||
return self.app.get_import_handler(self.import_handler_key, require=True)
|
||||
|
||||
# or maybe try spec lookup basd on setting class name
|
||||
class_name = self.__class__.__name__
|
||||
if match := re.match(
|
||||
r"^(?P<action>import|export)_to_(?P<target>\S+)_from_(?P<source>\S+)_warning$",
|
||||
class_name,
|
||||
):
|
||||
key = f"{match['action']}.to_{match['target']}.from_{match['source']}"
|
||||
return self.app.get_import_handler(key, require=True)
|
||||
|
||||
raise ValueError(
|
||||
"must set import_handler_spec (or import_handler_key) "
|
||||
f"for email setting: {class_name}"
|
||||
)
|
||||
|
||||
# nb. this is just used for sample data
|
||||
def make_diff(self, *args, **kwargs): # pylint: disable=missing-function-docstring
|
||||
return Diff(self.config, *args, **kwargs)
|
||||
|
||||
def sample_data(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
model = self.app.model
|
||||
handler = self.get_import_handler()
|
||||
|
||||
alice = model.User(username="alice")
|
||||
bob = model.User(username="bob")
|
||||
charlie = model.User(username="charlie")
|
||||
|
||||
runtime = datetime.timedelta(seconds=30)
|
||||
return {
|
||||
"handler": handler,
|
||||
"title": handler.get_title(),
|
||||
"source_title": handler.get_source_title(),
|
||||
"target_title": handler.get_target_title(),
|
||||
"runtime": runtime,
|
||||
"runtime_display": "30 seconds",
|
||||
"dry_run": True,
|
||||
"argv": [
|
||||
"bin/wutta",
|
||||
"import-foo",
|
||||
"User",
|
||||
"--delete",
|
||||
"--dry-run",
|
||||
"-W",
|
||||
],
|
||||
"changes": {
|
||||
"User": (
|
||||
[
|
||||
(
|
||||
alice,
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"),
|
||||
"username": "alice",
|
||||
},
|
||||
),
|
||||
],
|
||||
[
|
||||
(
|
||||
bob,
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
||||
"username": "bob",
|
||||
},
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
||||
"username": "bobbie",
|
||||
},
|
||||
),
|
||||
],
|
||||
[
|
||||
(
|
||||
charlie,
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"),
|
||||
"username": "charlie",
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
},
|
||||
"make_diff": self.make_diff,
|
||||
"max_diffs": 15,
|
||||
}
|
||||
|
||||
|
||||
class import_to_versions_from_wutta_warning( # pylint: disable=invalid-name
|
||||
ImportExportWarning
|
||||
):
|
||||
"""
|
||||
Diff warning for Wutta → Versions import.
|
||||
"""
|
||||
|
||||
|
||||
class import_to_wutta_from_csv_warning( # pylint: disable=invalid-name
|
||||
ImportExportWarning
|
||||
):
|
||||
"""
|
||||
Diff warning for CSV → Wutta import.
|
||||
"""
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
# Copyright © 2024 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
|
|
@ -32,7 +32,7 @@ And some :term:`import handler` base classes:
|
|||
* :class:`~wuttasync.importing.handlers.ImportHandler`
|
||||
* :class:`~wuttasync.importing.handlers.FromFileHandler`
|
||||
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
|
||||
* :class:`~wuttasync.importing.handlers.ToWuttaHandler`
|
||||
* :class:`~wuttasync.importing.wutta.ToWuttaHandler`
|
||||
|
||||
And some :term:`importer` base classes:
|
||||
|
||||
|
|
@ -42,12 +42,7 @@ And some :term:`importer` base classes:
|
|||
* :class:`~wuttasync.importing.model.ToWutta`
|
||||
"""
|
||||
|
||||
from .handlers import (
|
||||
Orientation,
|
||||
ImportHandler,
|
||||
FromFileHandler,
|
||||
ToSqlalchemyHandler,
|
||||
ToWuttaHandler,
|
||||
)
|
||||
from .handlers import Orientation, ImportHandler, FromFileHandler, ToSqlalchemyHandler
|
||||
from .base import Importer, FromFile, ToSqlalchemy
|
||||
from .model import ToWutta
|
||||
from .wutta import ToWuttaHandler
|
||||
|
|
|
|||
|
|
@ -184,19 +184,6 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
|||
:meth:`get_target_cache()`.
|
||||
"""
|
||||
|
||||
default_keys = None
|
||||
"""
|
||||
In certain edge cases, the importer class must declare its key
|
||||
list without using :attr:`keys`.
|
||||
|
||||
(As of now this only happens with
|
||||
:class:`~wuttasync.importing.versions.FromWuttaToVersions` which
|
||||
must dynamically create importer classes.)
|
||||
|
||||
If applicable, this value is used as fallback for
|
||||
:meth:`get_keys()`.
|
||||
"""
|
||||
|
||||
max_create = None
|
||||
max_update = None
|
||||
max_delete = None
|
||||
|
|
@ -336,54 +323,19 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
|||
|
||||
def get_keys(self):
|
||||
"""
|
||||
Retrieve the list of key field(s) for use with import/export.
|
||||
The result is cached, so the key list is only calculated once.
|
||||
|
||||
Many importers have just one key field, but we always assume a
|
||||
key *list* - so this often is a list with just one field.
|
||||
Must return the key field(s) for use with import/export.
|
||||
|
||||
All fields in this list should also be found in the output for
|
||||
:meth:`get_fields()`.
|
||||
|
||||
Many importers will declare this via :attr:`keys` (or
|
||||
:attr:`key`) static attribute::
|
||||
|
||||
class SprocketImporter(Importer):
|
||||
|
||||
# nb. all these examples work the same
|
||||
|
||||
# 'keys' is the preferred attribute
|
||||
keys = ("sprocket_id",) # <-- the "canonical" way
|
||||
keys = ["sprocket_id"]
|
||||
keys = "sprocket_id"
|
||||
|
||||
# 'key' is not preferred, but works
|
||||
key = ("sprocket_id",)
|
||||
key = "sprocket_id"
|
||||
|
||||
If neither ``keys`` nor ``key`` is set, as a special case
|
||||
:attr:`default_keys` is used if set.
|
||||
|
||||
If no keys were declared, the list is inspected from the model
|
||||
class via
|
||||
:func:`sqlalchemy-utils:sqlalchemy_utils.functions.get_primary_keys()`.
|
||||
|
||||
In any case, the determination is made only once. This method
|
||||
also *sets* :attr:`keys` on the instance, so it will return
|
||||
that as-is for subsequent calls.
|
||||
|
||||
:returns: List of "key" field names.
|
||||
"""
|
||||
keys = None
|
||||
|
||||
# nb. prefer 'keys' but use 'key' as fallback
|
||||
if "keys" in self.__dict__:
|
||||
keys = self.__dict__["keys"]
|
||||
elif "key" in self.__dict__:
|
||||
keys = self.__dict__["key"]
|
||||
else:
|
||||
keys = self.default_keys
|
||||
|
||||
if keys:
|
||||
if isinstance(keys, str):
|
||||
keys = self.config.parse_list(keys)
|
||||
|
|
@ -459,10 +411,7 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
|||
updated = []
|
||||
deleted = []
|
||||
|
||||
model_title = self.get_model_title()
|
||||
log.debug(
|
||||
"using key fields for %s: %s", model_title, ", ".join(self.get_keys())
|
||||
)
|
||||
log.debug("using key fields: %s", ", ".join(self.get_keys()))
|
||||
|
||||
# get complete set of normalized source data
|
||||
if source_data is None:
|
||||
|
|
@ -471,7 +420,8 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
|||
# nb. prune duplicate records from source data
|
||||
source_data, source_keys = self.get_unique_data(source_data)
|
||||
|
||||
log.debug("got %s %s records from source", len(source_data), model_title)
|
||||
model_title = self.get_model_title()
|
||||
log.debug(f"got %s {model_title} records from source", len(source_data))
|
||||
|
||||
# maybe cache existing target data
|
||||
if self.caches_target:
|
||||
|
|
@ -1321,139 +1271,10 @@ class FromFile(Importer):
|
|||
self.input_file.close()
|
||||
|
||||
|
||||
class QueryWrapper:
|
||||
"""
|
||||
Simple wrapper for a SQLAlchemy query, to make it sort of behave
|
||||
so that an importer can treat it as a data record list.
|
||||
|
||||
:param query: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
|
||||
"""
|
||||
|
||||
def __init__(self, query):
|
||||
self.query = query
|
||||
|
||||
def __len__(self):
|
||||
try:
|
||||
return len(self.query)
|
||||
except TypeError:
|
||||
return self.query.count()
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.query)
|
||||
|
||||
|
||||
class FromSqlalchemy(Importer): # pylint: disable=abstract-method
|
||||
"""
|
||||
Base class for importer/exporter using SQL/ORM query as data
|
||||
source.
|
||||
|
||||
Subclass should define :attr:`source_model_class` in which case
|
||||
the source query is automatic. And/or override
|
||||
:meth:`get_source_query()` to customize.
|
||||
|
||||
See also :class:`FromSqlalchemyMirror` and :class:`ToSqlalchemy`.
|
||||
"""
|
||||
|
||||
source_model_class = None
|
||||
"""
|
||||
Reference to the :term:`data model` class representing the source.
|
||||
|
||||
This normally is a SQLAlchemy mapped class, e.g.
|
||||
:class:`~wuttjamaican:wuttjamaican.db.model.base.Person` for
|
||||
exporting from the Wutta People table.
|
||||
"""
|
||||
|
||||
source_session = None
|
||||
"""
|
||||
Reference to the open :term:`db session` for the data source.
|
||||
|
||||
The importer must be given this reference when instantiated by the
|
||||
:term:`import handler`. This is handled automatically if using
|
||||
:class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`.
|
||||
"""
|
||||
|
||||
def get_source_objects(self):
|
||||
"""
|
||||
This method is responsible for fetching "raw" (non-normalized)
|
||||
records from data source.
|
||||
|
||||
(See also the parent method docs for
|
||||
:meth:`~wuttasync.importing.base.Importer.get_source_objects()`.)
|
||||
|
||||
It calls :meth:`get_source_query()` and then wraps that in a
|
||||
:class:`QueryWrapper`, which is then returned.
|
||||
|
||||
Note that this method does not technically "retrieve" records
|
||||
from the query; that happens automatically later.
|
||||
|
||||
:returns: :class:`QueryWrapper` for the source query
|
||||
"""
|
||||
query = self.get_source_query()
|
||||
return QueryWrapper(query)
|
||||
|
||||
def get_source_query(self):
|
||||
"""
|
||||
This returns the SQL/ORM query used to fetch source
|
||||
data. It is called from :meth:`get_source_objects()`.
|
||||
|
||||
Default logic just makes a simple ``SELECT * FROM TABLE`` kind
|
||||
of query. Subclass can override as needed.
|
||||
|
||||
:returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
|
||||
"""
|
||||
return self.source_session.query(self.source_model_class)
|
||||
|
||||
|
||||
class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method
|
||||
"""
|
||||
Special base class for when the source and target are effectively
|
||||
mirrored, and can each be represented by the same :term:`data
|
||||
model`.
|
||||
|
||||
The assumption is that SQLAlchemy ORM is used on both sides, even
|
||||
though this base class only defines the source side (it inherits
|
||||
from :class:`FromSqlalchemy`).
|
||||
|
||||
There are two main use cases for this:
|
||||
|
||||
* sync between app nodes
|
||||
* sync version tables
|
||||
|
||||
When 2 app nodes are synced, the source and target are "the same"
|
||||
in a schema sense, e.g. ``sprockets on node 01 => sprockets on
|
||||
node 02``.
|
||||
|
||||
When version tables are synced, the same schema can be used for
|
||||
the "live" table and the "version" table, e.g. ``sprockets =>
|
||||
sprocket versions``.
|
||||
"""
|
||||
|
||||
@property
|
||||
def source_model_class(self):
|
||||
"""
|
||||
This returns the :attr:`~Importer.model_class` since source
|
||||
and target must share common schema.
|
||||
"""
|
||||
return self.model_class
|
||||
|
||||
def normalize_source_object(self, obj):
|
||||
"""
|
||||
Since source/target share schema, there should be no tricky
|
||||
normalization involved.
|
||||
|
||||
This calls :meth:`~Importer.normalize_target_object()` since
|
||||
that logic should already be defined. This ensures the same
|
||||
normalization is used on both sides.
|
||||
"""
|
||||
return self.normalize_target_object(obj)
|
||||
|
||||
|
||||
class ToSqlalchemy(Importer):
|
||||
"""
|
||||
Base class for importer/exporter which uses SQLAlchemy ORM on the
|
||||
target side.
|
||||
|
||||
See also :class:`FromSqlalchemy`.
|
||||
"""
|
||||
|
||||
caches_target = True
|
||||
|
|
@ -1491,8 +1312,6 @@ class ToSqlalchemy(Importer):
|
|||
Returns an ORM query suitable to fetch existing objects from
|
||||
the target side. This is called from
|
||||
:meth:`get_target_objects()`.
|
||||
|
||||
:returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
|
||||
"""
|
||||
return self.target_session.query(self.model_class)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,8 @@ from wuttjamaican.db.util import make_topo_sortkey, UUID
|
|||
from wuttjamaican.util import parse_bool
|
||||
|
||||
from .base import FromFile
|
||||
from .handlers import FromFileHandler, ToWuttaHandler
|
||||
from .handlers import FromFileHandler
|
||||
from .wutta import ToWuttaHandler
|
||||
from .model import ToWutta
|
||||
|
||||
|
||||
|
|
@ -238,8 +239,6 @@ class FromCsvToSqlalchemyHandlerMixin:
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# TODO: pylint (correctly) flags this as duplicate code, matching
|
||||
# on the wuttasync.importing.versions module - should fix?
|
||||
def define_importers(self):
|
||||
"""
|
||||
This mixin overrides typical (manual) importer definition, and
|
||||
|
|
@ -253,7 +252,6 @@ class FromCsvToSqlalchemyHandlerMixin:
|
|||
importers = {}
|
||||
model = self.get_target_model()
|
||||
|
||||
# pylint: disable=duplicate-code
|
||||
# mostly try to make an importer for every data model
|
||||
for name in dir(model):
|
||||
cls = getattr(model, name)
|
||||
|
|
|
|||
|
|
@ -26,14 +26,10 @@ Data Import / Export Handlers
|
|||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from enum import Enum
|
||||
|
||||
import humanize
|
||||
|
||||
from wuttjamaican.app import GenericHandler
|
||||
from wuttjamaican.diffs import Diff
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -48,7 +44,7 @@ class Orientation(Enum):
|
|||
EXPORT = "export"
|
||||
|
||||
|
||||
class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||
class ImportHandler(GenericHandler):
|
||||
"""
|
||||
Base class for all import/export handlers.
|
||||
|
||||
|
|
@ -125,47 +121,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
:meth:`commit_transaction()`.
|
||||
"""
|
||||
|
||||
process_started = None
|
||||
|
||||
warnings = False
|
||||
"""
|
||||
Boolean indicating the import/export should run in "warnings"
|
||||
mode.
|
||||
|
||||
If set, this declares that no changes are expected for the
|
||||
import/export job. If any changes do occur with this flag set, a
|
||||
diff warning email is sent within :meth:`process_changes()`.
|
||||
|
||||
See also :attr:`warnings_recipients`,
|
||||
:attr:`warnings_max_diffs` and :attr:`warnings_email_key`.
|
||||
"""
|
||||
|
||||
warnings_email_key = None
|
||||
"""
|
||||
Explicit :term:`email key` for sending the diff warning email,
|
||||
*unique to this import/export type*.
|
||||
|
||||
Handlers do not normally set this, so the email key is determined
|
||||
automatically within :meth:`get_warnings_email_key()`.
|
||||
|
||||
See also :attr:`warnings`.
|
||||
"""
|
||||
|
||||
warnings_recipients = None
|
||||
"""
|
||||
Explicit recipient list for the warning email. If not set, the
|
||||
recipients are determined automatically via config.
|
||||
|
||||
See also :attr:`warnings`.
|
||||
"""
|
||||
|
||||
warnings_max_diffs = 15
|
||||
"""
|
||||
Max number of record diffs (per model) to show in the warning email.
|
||||
|
||||
See also :attr:`warnings`.
|
||||
"""
|
||||
|
||||
importers = None
|
||||
"""
|
||||
This should be a dict of all importer/exporter classes available
|
||||
|
|
@ -209,21 +164,18 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
@classmethod
|
||||
def get_key(cls):
|
||||
"""
|
||||
Returns the :term:`import/export key` for the handler. This
|
||||
is a combination of :attr:`source_key` and :attr:`target_key`
|
||||
and :attr:`orientation`.
|
||||
Returns the "full key" for the handler. This is a combination
|
||||
of :attr:`source_key` and :attr:`target_key` and
|
||||
:attr:`orientation`.
|
||||
|
||||
For instance in the case of Wutta → CSV export, the key is:
|
||||
``export.to_csv.from_wutta``
|
||||
For instance in the case of CSV → Wutta, the full handler key
|
||||
is ``to_wutta.from_csv.import``.
|
||||
|
||||
Note that more than one handler may use the same key; but only
|
||||
one will be configured as the "designated" handler for that
|
||||
key, a la
|
||||
:meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`.
|
||||
|
||||
See also :meth:`get_spec()`.
|
||||
Note that more than one handler may return the same full key
|
||||
here; but only one will be configured as the "default" handler
|
||||
for that key. See also :meth:`get_spec()`.
|
||||
"""
|
||||
return f"{cls.orientation.value}.to_{cls.target_key}.from_{cls.source_key}"
|
||||
return f"to_{cls.target_key}.from_{cls.source_key}.{cls.orientation.value}"
|
||||
|
||||
@classmethod
|
||||
def get_spec(cls):
|
||||
|
|
@ -257,20 +209,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
"""
|
||||
Returns the display title for the data source.
|
||||
|
||||
By default this returns :attr:`source_key`, but this can be
|
||||
overriden by class attribute.
|
||||
|
||||
Base class can define ``generic_source_title`` to provide a
|
||||
new default::
|
||||
|
||||
class FromExcelHandler(ImportHandler):
|
||||
generic_source_title = "Excel File"
|
||||
|
||||
Subclass can define ``source_title`` to be explicit::
|
||||
|
||||
class FromExcelToWutta(FromExcelHandler, ToWuttaHandler):
|
||||
source_title = "My Spreadsheet"
|
||||
|
||||
See also :meth:`get_title()` and :meth:`get_target_title()`.
|
||||
"""
|
||||
if hasattr(self, "source_title"):
|
||||
|
|
@ -283,20 +221,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
"""
|
||||
Returns the display title for the data target.
|
||||
|
||||
By default this returns :attr:`target_key`, but this can be
|
||||
overriden by class attribute.
|
||||
|
||||
Base class can define ``generic_target_title`` to provide a
|
||||
new default::
|
||||
|
||||
class ToExcelHandler(ImportHandler):
|
||||
generic_target_title = "Excel File"
|
||||
|
||||
Subclass can define ``target_title`` to be explicit::
|
||||
|
||||
class FromWuttaToExcel(FromWuttaHandler, ToExcelHandler):
|
||||
target_title = "My Spreadsheet"
|
||||
|
||||
See also :meth:`get_title()` and :meth:`get_source_title()`.
|
||||
"""
|
||||
if hasattr(self, "target_title"):
|
||||
|
|
@ -326,17 +250,11 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
* :meth:`begin_transaction()`
|
||||
* :meth:`get_importer()`
|
||||
* :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter)
|
||||
* :meth:`process_changes()`
|
||||
* :meth:`rollback_transaction()`
|
||||
* :meth:`commit_transaction()`
|
||||
"""
|
||||
kwargs = self.consume_kwargs(kwargs)
|
||||
self.process_started = self.app.localtime()
|
||||
self.begin_transaction()
|
||||
changes = OrderedDict()
|
||||
|
||||
if not keys:
|
||||
keys = list(self.importers)
|
||||
|
||||
success = False
|
||||
try:
|
||||
|
|
@ -347,31 +265,22 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
# invoke importer
|
||||
importer = self.get_importer(key, **kwargs)
|
||||
created, updated, deleted = importer.process_data()
|
||||
changed = bool(created or updated or deleted)
|
||||
|
||||
# log what happened
|
||||
msg = "%s: added %d; updated %d; deleted %d %s records"
|
||||
if self.dry_run:
|
||||
msg += " (dry run)"
|
||||
logger = log.warning if changed and self.warnings else log.info
|
||||
logger(
|
||||
log.info(
|
||||
msg, self.get_title(), len(created), len(updated), len(deleted), key
|
||||
)
|
||||
|
||||
# keep track of any changes
|
||||
if changed:
|
||||
changes[key] = created, updated, deleted
|
||||
|
||||
# post-processing for all changes
|
||||
if changes:
|
||||
self.process_changes(changes)
|
||||
|
||||
success = True
|
||||
|
||||
except:
|
||||
log.exception("what should happen here?") # TODO
|
||||
raise
|
||||
|
||||
else:
|
||||
success = True
|
||||
|
||||
finally:
|
||||
if not success:
|
||||
log.warning("something failed, so transaction was rolled back")
|
||||
|
|
@ -405,17 +314,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
if "dry_run" in kwargs:
|
||||
self.dry_run = kwargs["dry_run"]
|
||||
|
||||
if "warnings" in kwargs:
|
||||
self.warnings = kwargs.pop("warnings")
|
||||
|
||||
if "warnings_recipients" in kwargs:
|
||||
self.warnings_recipients = self.config.parse_list(
|
||||
kwargs.pop("warnings_recipients")
|
||||
)
|
||||
|
||||
if "warnings_max_diffs" in kwargs:
|
||||
self.warnings_max_diffs = kwargs.pop("warnings_max_diffs")
|
||||
|
||||
return kwargs
|
||||
|
||||
def begin_transaction(self):
|
||||
|
|
@ -614,113 +512,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
"""
|
||||
return kwargs
|
||||
|
||||
def process_changes(self, changes):
|
||||
"""
|
||||
Run post-processing operations on the given changes, if
|
||||
applicable.
|
||||
|
||||
This method is called by :meth:`process_data()`, if any
|
||||
changes were made.
|
||||
|
||||
Default logic will send a "diff warning" email to the
|
||||
configured recipient(s), if :attr:`warnings` mode is enabled.
|
||||
If it is not enabled, nothing happens.
|
||||
|
||||
:param changes: :class:`~python:collections.OrderedDict` of
|
||||
changes from the overall import/export job. The structure
|
||||
is described below.
|
||||
|
||||
Keys for the ``changes`` dict will be model/importer names,
|
||||
for instance::
|
||||
|
||||
{
|
||||
"Sprocket": {...},
|
||||
"User": {...},
|
||||
}
|
||||
|
||||
Value for each model key is a 3-tuple of ``(created, updated,
|
||||
deleted)``. Each of those elements is a list::
|
||||
|
||||
{
|
||||
"Sprocket": (
|
||||
[...], # created
|
||||
[...], # updated
|
||||
[...], # deleted
|
||||
),
|
||||
}
|
||||
|
||||
The list elements are always tuples, but the structure
|
||||
varies::
|
||||
|
||||
{
|
||||
"Sprocket": (
|
||||
[ # created, 2-tuples
|
||||
(obj, source_data),
|
||||
],
|
||||
[ # updated, 3-tuples
|
||||
(obj, source_data, target_data),
|
||||
],
|
||||
[ # deleted, 2-tuples
|
||||
(obj, target_data),
|
||||
],
|
||||
),
|
||||
}
|
||||
"""
|
||||
if not self.warnings:
|
||||
return
|
||||
|
||||
def make_diff(*args, **kwargs):
|
||||
return Diff(self.config, *args, **kwargs)
|
||||
|
||||
runtime = self.app.localtime() - self.process_started
|
||||
data = {
|
||||
"handler": self,
|
||||
"title": self.get_title(),
|
||||
"source_title": self.get_source_title(),
|
||||
"target_title": self.get_target_title(),
|
||||
"dry_run": self.dry_run,
|
||||
"argv": sys.argv,
|
||||
"runtime": runtime,
|
||||
"runtime_display": humanize.naturaldelta(runtime),
|
||||
"changes": changes,
|
||||
"make_diff": make_diff,
|
||||
"max_diffs": self.warnings_max_diffs,
|
||||
}
|
||||
|
||||
# maybe override recipients
|
||||
kw = {}
|
||||
if self.warnings_recipients:
|
||||
kw["to"] = self.warnings_recipients
|
||||
# TODO: should we in fact clear these..?
|
||||
kw["cc"] = []
|
||||
kw["bcc"] = []
|
||||
|
||||
# send the email
|
||||
email_key = self.get_warnings_email_key()
|
||||
self.app.send_email(email_key, data, fallback_key="import_export_warning", **kw)
|
||||
|
||||
log.info("%s: warning email was sent", self.get_title())
|
||||
|
||||
def get_warnings_email_key(self):
|
||||
"""
|
||||
Returns the :term:`email key` to be used for sending the diff
|
||||
warning email.
|
||||
|
||||
The email key should be unique to this import/export type
|
||||
(really, the :term:`import/export key`) but not necessarily
|
||||
unique to one handler.
|
||||
|
||||
If :attr:`warnings_email_key` is set, it will be used as-is.
|
||||
|
||||
Otherwise one is generated from :meth:`get_key()`.
|
||||
|
||||
:returns: Email key for diff warnings
|
||||
"""
|
||||
if self.warnings_email_key:
|
||||
return self.warnings_email_key
|
||||
|
||||
return self.get_key().replace(".", "_") + "_warning"
|
||||
|
||||
|
||||
class FromFileHandler(ImportHandler):
|
||||
"""
|
||||
|
|
@ -747,129 +538,9 @@ class FromFileHandler(ImportHandler):
|
|||
super().process_data(*keys, **kwargs)
|
||||
|
||||
|
||||
class FromSqlalchemyHandler(ImportHandler):
|
||||
"""
|
||||
Base class for import/export handlers using SQLAlchemy ORM (DB) as
|
||||
data source.
|
||||
|
||||
This is meant to be used with importers/exporters which inherit
|
||||
from :class:`~wuttasync.importing.base.FromSqlalchemy`. It will
|
||||
set the
|
||||
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_session`
|
||||
attribute when making them; cf. :meth:`get_importer_kwargs()`.
|
||||
|
||||
This is the base class for :class:`FromWuttaHandler`, but can be
|
||||
used with any database.
|
||||
|
||||
See also :class:`ToSqlalchemyHandler`.
|
||||
"""
|
||||
|
||||
source_session = None
|
||||
"""
|
||||
Reference to the :term:`db session` for data source.
|
||||
|
||||
This will be ``None`` unless a transaction is running.
|
||||
"""
|
||||
|
||||
def begin_source_transaction(self):
|
||||
"""
|
||||
This calls :meth:`make_source_session()` and assigns the
|
||||
result to :attr:`source_session`.
|
||||
"""
|
||||
self.source_session = self.make_source_session()
|
||||
|
||||
def commit_source_transaction(self):
|
||||
"""
|
||||
This commits and closes :attr:`source_session`.
|
||||
"""
|
||||
self.source_session.commit()
|
||||
self.source_session.close()
|
||||
self.source_session = None
|
||||
|
||||
def rollback_source_transaction(self):
|
||||
"""
|
||||
This rolls back, then closes :attr:`source_session`.
|
||||
"""
|
||||
self.source_session.rollback()
|
||||
self.source_session.close()
|
||||
self.source_session = None
|
||||
|
||||
def make_source_session(self):
|
||||
"""
|
||||
Make and return a new :term:`db session` for the data source.
|
||||
|
||||
Default logic is not implemented; subclass must override.
|
||||
|
||||
:returns: :class:`~sqlalchemy.orm.Session` instance
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_importer_kwargs(self, key, **kwargs):
|
||||
"""
|
||||
This modifies the new importer kwargs to add:
|
||||
|
||||
* ``source_session`` - reference to :attr:`source_session`
|
||||
|
||||
See also docs for parent method,
|
||||
:meth:`~ImportHandler.get_importer_kwargs()`.
|
||||
"""
|
||||
kwargs = super().get_importer_kwargs(key, **kwargs)
|
||||
kwargs["source_session"] = self.source_session
|
||||
return kwargs
|
||||
|
||||
|
||||
class FromWuttaHandler(FromSqlalchemyHandler):
|
||||
"""
|
||||
Handler for import/export which uses Wutta ORM (:term:`app
|
||||
database`) as data source.
|
||||
|
||||
This inherits from :class:`FromSqlalchemyHandler`.
|
||||
|
||||
See also :class:`ToWuttaHandler`.
|
||||
"""
|
||||
|
||||
source_key = "wutta"
|
||||
"" # nb. suppress docs
|
||||
|
||||
def get_source_title(self):
|
||||
"""
|
||||
This overrides default logic to use
|
||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.get_title()`
|
||||
as the default value.
|
||||
|
||||
Subclass can still define
|
||||
:attr:`~wuttasync.importing.handlers.ImportHandler.source_title`
|
||||
(or
|
||||
:attr:`~wuttasync.importing.handlers.ImportHandler.generic_source_title`)
|
||||
to customize.
|
||||
|
||||
See also docs for parent method:
|
||||
:meth:`~wuttasync.importing.handlers.ImportHandler.get_source_title()`
|
||||
"""
|
||||
if hasattr(self, "source_title"):
|
||||
return self.source_title
|
||||
if hasattr(self, "generic_source_title"):
|
||||
return self.generic_source_title
|
||||
return self.app.get_title()
|
||||
|
||||
def make_source_session(self):
|
||||
"""
|
||||
This calls
|
||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
||||
and returns it.
|
||||
"""
|
||||
return self.app.make_session()
|
||||
|
||||
|
||||
class ToSqlalchemyHandler(ImportHandler):
|
||||
"""
|
||||
Base class for import/export handlers which target a SQLAlchemy
|
||||
ORM (DB).
|
||||
|
||||
This is the base class for :class:`ToWuttaHandler`, but can be
|
||||
used with any database.
|
||||
|
||||
See also :class:`FromSqlalchemyHandler`.
|
||||
Handler for import/export which targets a SQLAlchemy ORM (DB).
|
||||
"""
|
||||
|
||||
target_session = None
|
||||
|
|
@ -920,37 +591,3 @@ class ToSqlalchemyHandler(ImportHandler):
|
|||
kwargs = super().get_importer_kwargs(key, **kwargs)
|
||||
kwargs.setdefault("target_session", self.target_session)
|
||||
return kwargs
|
||||
|
||||
|
||||
class ToWuttaHandler(ToSqlalchemyHandler):
|
||||
"""
|
||||
Handler for import/export which targets Wutta ORM (:term:`app
|
||||
database`).
|
||||
|
||||
This inherits from :class:`ToSqlalchemyHandler`.
|
||||
|
||||
See also :class:`FromWuttaHandler`.
|
||||
"""
|
||||
|
||||
target_key = "wutta"
|
||||
"" # nb. suppress docs
|
||||
|
||||
def get_target_title(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
# nb. we override parent to use app title as default
|
||||
if hasattr(self, "target_title"):
|
||||
return self.target_title
|
||||
if hasattr(self, "generic_target_title"):
|
||||
return self.generic_target_title
|
||||
return self.app.get_title()
|
||||
|
||||
def make_target_session(self):
|
||||
"""
|
||||
Call
|
||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
||||
and return the result.
|
||||
|
||||
:returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
|
||||
instance.
|
||||
"""
|
||||
return self.app.make_session()
|
||||
|
|
|
|||
|
|
@ -1,353 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Importing Versions
|
||||
|
||||
This is a special type of import, only relevant when data versioning
|
||||
is enabled.
|
||||
|
||||
See the handler class for more info: :class:`FromWuttaToVersions`
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from sqlalchemy_utils.functions import get_primary_keys
|
||||
|
||||
from wuttjamaican.db.util import make_topo_sortkey
|
||||
|
||||
from .handlers import FromWuttaHandler, ToWuttaHandler
|
||||
from .wutta import FromWuttaMirror
|
||||
from .model import ToWutta
|
||||
|
||||
|
||||
class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
|
||||
"""
|
||||
Handler for Wutta -> Versions import.
|
||||
|
||||
The purpose of this is to ensure version tables accurately reflect
|
||||
the current "live" data set, for given table(s). It is only
|
||||
relevant/usable if versioning is configured and enabled. For more
|
||||
on that see :doc:`wutta-continuum:index`.
|
||||
|
||||
For a given import model, the source is the "live" table, target
|
||||
is the "version" table - both in the same :term:`app database`.
|
||||
|
||||
When reading data from the target side, it only grabs the "latest"
|
||||
(valid) version record for each comparison to source.
|
||||
|
||||
When changes are needed, instead of updating the existing version
|
||||
record, it always writes a new version record.
|
||||
|
||||
This handler will dynamically create importers for all versioned
|
||||
models in the :term:`app model`; see
|
||||
:meth:`make_importer_factory()`.
|
||||
"""
|
||||
|
||||
target_key = "versions"
|
||||
target_title = "Versions"
|
||||
|
||||
continuum_uow = None
|
||||
"""
|
||||
Reference to the
|
||||
:class:`sqlalchemy-continuum:`sqlalchemy_continuum.UnitOfWork`
|
||||
created (by the SQLAlchemy-Continuum ``versioning_manager``) when
|
||||
the transaction begins.
|
||||
|
||||
See also :attr:`continuum_txn` and
|
||||
:meth:`begin_target_transaction()`.
|
||||
"""
|
||||
|
||||
continuum_txn = None
|
||||
"""
|
||||
Reference to the SQLAlchemy-Continuum ``transaction`` record, to
|
||||
which any new version records will associate (if needed).
|
||||
|
||||
This transaction will track the effective user responsible for
|
||||
the change(s), their client IP, and timestamp.
|
||||
|
||||
This reference is passed along to the importers as well (as
|
||||
:attr:`~FromWuttaToVersionBase.continuum_txn`) via
|
||||
:meth:`get_importer_kwargs()`.
|
||||
|
||||
See also :attr:`continuum_uow`.
|
||||
"""
|
||||
|
||||
continuum_comment = None
|
||||
|
||||
def consume_kwargs(self, kwargs):
|
||||
kwargs = super().consume_kwargs(kwargs)
|
||||
self.continuum_comment = kwargs.pop("comment", None)
|
||||
return kwargs
|
||||
|
||||
def begin_target_transaction(self):
|
||||
# pylint: disable=line-too-long
|
||||
"""
|
||||
In addition to normal logic, this does some setup for
|
||||
SQLAlchemy-Continuum:
|
||||
|
||||
It establishes a "unit of work" by calling
|
||||
:meth:`~sqlalchemy-continuum:sqlalchemy_continuum.VersioningManager.unit_of_work()`,
|
||||
assigning the result to :attr:`continuum_uow`.
|
||||
|
||||
It then calls
|
||||
:meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()`
|
||||
and assigns that to :attr:`continuum_txn`.
|
||||
|
||||
It also sets the comment for the transaction, if applicable.
|
||||
|
||||
See also docs for parent method:
|
||||
:meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()`
|
||||
"""
|
||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
||||
|
||||
super().begin_target_transaction()
|
||||
|
||||
self.continuum_uow = continuum.versioning_manager.unit_of_work(
|
||||
self.target_session
|
||||
)
|
||||
|
||||
self.continuum_txn = self.continuum_uow.create_transaction(self.target_session)
|
||||
|
||||
if self.continuum_comment:
|
||||
self.continuum_txn.meta = {"comment": self.continuum_comment}
|
||||
|
||||
def get_importer_kwargs(self, key, **kwargs):
|
||||
"""
|
||||
This modifies the new importer kwargs to add:
|
||||
|
||||
* ``continuum_txn`` - reference to :attr:`continuum_txn`
|
||||
|
||||
See also docs for parent method:
|
||||
:meth:`~wuttasync.importing.handlers.ImportHandler.get_importer_kwargs()`
|
||||
"""
|
||||
kwargs = super().get_importer_kwargs(key, **kwargs)
|
||||
kwargs["continuum_txn"] = self.continuum_txn
|
||||
return kwargs
|
||||
|
||||
# TODO: pylint (correctly) flags this as duplicate code, matching
|
||||
# on the wuttasync.importing.csv module - should fix?
|
||||
def define_importers(self):
|
||||
"""
|
||||
This overrides typical (manual) importer definition, instead
|
||||
generating importers for all versioned models.
|
||||
|
||||
It will inspect the :term:`app model` and call
|
||||
:meth:`make_importer_factory()` for each model found, keeping
|
||||
only the valid importers.
|
||||
|
||||
See also the docs for parent method:
|
||||
:meth:`~wuttasync.importing.handlers.ImportHandler.define_importers()`
|
||||
"""
|
||||
model = self.app.model
|
||||
importers = {}
|
||||
|
||||
# pylint: disable=duplicate-code
|
||||
# mostly try to make an importer for every data model
|
||||
for name in dir(model):
|
||||
cls = getattr(model, name)
|
||||
if (
|
||||
isinstance(cls, type)
|
||||
and issubclass(cls, model.Base)
|
||||
and cls is not model.Base
|
||||
):
|
||||
# only keep "good" importers, i.e. for versioned models
|
||||
if factory := self.make_importer_factory(cls, name):
|
||||
importers[name] = factory
|
||||
|
||||
# sort importers according to schema topography
|
||||
topo_sortkey = make_topo_sortkey(model)
|
||||
importers = OrderedDict(
|
||||
[(name, importers[name]) for name in sorted(importers, key=topo_sortkey)]
|
||||
)
|
||||
|
||||
return importers
|
||||
|
||||
def make_importer_factory(self, model_class, name):
|
||||
"""
|
||||
Try to generate a new :term:`importer` class for the given
|
||||
:term:`data model`. This is called by
|
||||
:meth:`define_importers()`.
|
||||
|
||||
If the provided ``model_class`` is not versioned, this will
|
||||
fail and return ``None``.
|
||||
|
||||
For a versioned model, the new importer class will inherit
|
||||
from :class:`FromWuttaToVersionBase`.
|
||||
|
||||
Its (target)
|
||||
:attr:`~wuttasync.importing.base.Importer.model_class` will be
|
||||
set to the **version** model.
|
||||
|
||||
Its
|
||||
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class`
|
||||
will be set to the **normal** model.
|
||||
|
||||
:param model_class: A (normal, not version) data model class.
|
||||
|
||||
:param name: The "model name" for the importer. New class
|
||||
name will be based on this, so e.g. ``Widget`` model name
|
||||
becomes ``WidgetImporter`` class name.
|
||||
|
||||
:returns: The new class, or ``None``
|
||||
"""
|
||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
||||
|
||||
try:
|
||||
version_class = continuum.version_class(model_class)
|
||||
except continuum.exc.ClassNotVersioned:
|
||||
return None
|
||||
|
||||
return type(
|
||||
f"{name}Importer",
|
||||
(FromWuttaToVersionBase,),
|
||||
{
|
||||
"source_model_class": model_class,
|
||||
"model_class": version_class,
|
||||
"default_keys": list(get_primary_keys(model_class)),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class FromWuttaToVersionBase(FromWuttaMirror, ToWutta):
|
||||
"""
|
||||
Base importer class for Wutta -> Versions.
|
||||
|
||||
This imports from
|
||||
:class:`~wuttasync.importing.wutta.FromWuttaMirror` and
|
||||
:class:`~wuttasync.importing.model.ToWutta`.
|
||||
|
||||
The import handler will dynamically generate importers using this
|
||||
base class; see
|
||||
:meth:`~FromWuttaToVersions.make_importer_factory()`.
|
||||
"""
|
||||
|
||||
continuum_txn = None
|
||||
"""
|
||||
Reference to the handler's attribute of the same name:
|
||||
:attr:`~FromWuttaToVersions.continuum_txn`
|
||||
|
||||
This is the SQLAlchemy-Continuum ``transaction`` record, to which
|
||||
any new version records will associate (if needed).
|
||||
|
||||
This transaction will track the effective user responsible for
|
||||
the change(s), their client IP, and timestamp.
|
||||
"""
|
||||
|
||||
def get_simple_fields(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
fields = super().get_simple_fields()
|
||||
unwanted = ["transaction_id", "operation_type", "end_transaction_id"]
|
||||
fields = [field for field in fields if field not in unwanted]
|
||||
return fields
|
||||
|
||||
def get_target_query(self, source_data=None):
|
||||
"""
|
||||
This modifies the normal query to ensure we only get the
|
||||
"latest valid" version for each record, for comparison to
|
||||
source.
|
||||
|
||||
.. note::
|
||||
|
||||
In some cases, it still may be possible for multiple
|
||||
"latest" versions to match for a given record. This means
|
||||
inconsistent data; a warning should be logged if so, and
|
||||
you must track it down...
|
||||
|
||||
See also docs for parent method:
|
||||
:meth:`~wuttasync.importing.base.ToSqlalchemy.get_target_query()`
|
||||
"""
|
||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
||||
|
||||
# pylint: disable=singleton-comparison
|
||||
return (
|
||||
self.target_session.query(self.model_class)
|
||||
.filter(self.model_class.end_transaction_id == None)
|
||||
.filter(self.model_class.operation_type != continuum.Operation.DELETE)
|
||||
)
|
||||
|
||||
def normalize_target_object(self, obj): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
data = super().normalize_target_object(obj)
|
||||
|
||||
# we want to add the original version object to normalized
|
||||
# data, so we can access it later for updating if needed. but
|
||||
# this method is called for *both* sides (source+target) since
|
||||
# this is a "mirrored" importer. so we must check the type
|
||||
# and only cache true versions, ignore "normal" objects.
|
||||
if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type
|
||||
obj, self.model_class
|
||||
):
|
||||
data["_objref"] = obj
|
||||
|
||||
return data
|
||||
|
||||
def make_version( # pylint: disable=missing-function-docstring
|
||||
self, source_data, operation_type
|
||||
):
|
||||
key = self.get_record_key(source_data)
|
||||
with self.target_session.no_autoflush:
|
||||
version = self.make_empty_object(key)
|
||||
self.populate(version, source_data)
|
||||
version.transaction = self.continuum_txn
|
||||
version.operation_type = operation_type
|
||||
self.target_session.add(version)
|
||||
return version
|
||||
|
||||
def populate(self, obj, data): # pylint: disable=missing-function-docstring
|
||||
keys = self.get_keys()
|
||||
for field in self.get_simple_fields():
|
||||
if field not in keys and field in data and field in self.fields:
|
||||
setattr(obj, field, data[field])
|
||||
|
||||
def create_target_object(self, key, source_data): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
||||
|
||||
return self.make_version(source_data, continuum.Operation.INSERT)
|
||||
|
||||
def update_target_object( # pylint: disable=empty-docstring
|
||||
self, obj, source_data, target_data=None
|
||||
):
|
||||
""" """
|
||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
||||
|
||||
# when we "update" it always involves making a *new* version
|
||||
# record. but that requires actually updating the "previous"
|
||||
# version to indicate the new version's transaction.
|
||||
prev_version = target_data.pop("_objref")
|
||||
prev_version.end_transaction_id = self.continuum_txn.id
|
||||
|
||||
return self.make_version(source_data, continuum.Operation.UPDATE)
|
||||
|
||||
def delete_target_object(self, obj): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
||||
|
||||
# nb. `obj` here is the existing/old version record; we update
|
||||
# it to indicate the new version's transaction.
|
||||
obj.end_transaction_id = self.continuum_txn.id
|
||||
|
||||
# add new "DELETE" version record. values should be the same as
|
||||
# for "previous" (existing/old) version.
|
||||
source_data = self.normalize_target_object(obj)
|
||||
return self.make_version(source_data, continuum.Operation.DELETE)
|
||||
|
|
@ -21,13 +21,37 @@
|
|||
#
|
||||
################################################################################
|
||||
"""
|
||||
Wutta → Wutta import/export
|
||||
Wutta ⇄ Wutta import/export
|
||||
"""
|
||||
|
||||
from .base import FromSqlalchemyMirror
|
||||
from .handlers import ToSqlalchemyHandler
|
||||
|
||||
|
||||
class FromWuttaMirror(FromSqlalchemyMirror): # pylint: disable=abstract-method
|
||||
class ToWuttaHandler(ToSqlalchemyHandler):
|
||||
"""
|
||||
Base class for Wutta -> Wutta data importers.
|
||||
Handler for import/export which targets Wutta ORM (:term:`app
|
||||
database`).
|
||||
"""
|
||||
|
||||
target_key = "wutta"
|
||||
"" # nb. suppress docs
|
||||
|
||||
def get_target_title(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
# nb. we override parent to use app title as default
|
||||
if hasattr(self, "target_title"):
|
||||
return self.target_title
|
||||
if hasattr(self, "generic_target_title"):
|
||||
return self.generic_target_title
|
||||
return self.app.get_title()
|
||||
|
||||
def make_target_session(self):
|
||||
"""
|
||||
Call
|
||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
||||
and return the result.
|
||||
|
||||
:returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
|
||||
instance.
|
||||
"""
|
||||
return self.app.make_session()
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Testing utilities
|
||||
"""
|
||||
|
||||
from wuttjamaican.testing import ConfigTestCase
|
||||
|
||||
|
||||
class ImportExportWarningTestCase(ConfigTestCase):
|
||||
"""
|
||||
Base class for testing the import/export warning email settings.
|
||||
|
||||
This inherits from
|
||||
:class:`~wuttjamaican:wuttjamaican.testing.ConfigTestCase`.
|
||||
|
||||
Example usage::
|
||||
|
||||
from wuttasync.testing import ImportExportWarningTestCase
|
||||
|
||||
class TestEmailSettings(ImportExportWarningTestCase):
|
||||
|
||||
def test_import_to_wutta_from_foo_warning(self):
|
||||
self.do_test_preview("import_to_wutta_from_foo_warning")
|
||||
|
||||
def test_export_to_foo_from_wutta_warning(self):
|
||||
self.do_test_preview("export_to_foo_from_wutta_warning")
|
||||
"""
|
||||
|
||||
app_title = "Wutta Poser"
|
||||
|
||||
def setUp(self):
|
||||
self.setup_config()
|
||||
self.config.setdefault("wutta.app_title", self.app_title)
|
||||
|
||||
def make_preview( # pylint: disable=missing-function-docstring,unused-argument
|
||||
self, key, mode="html"
|
||||
):
|
||||
handler = self.app.get_email_handler()
|
||||
setting = handler.get_email_setting(key)
|
||||
context = setting.sample_data()
|
||||
return handler.get_auto_html_body(
|
||||
setting.key, context, fallback_key=setting.fallback_key
|
||||
)
|
||||
|
||||
def do_test_preview(self, key): # pylint: disable=missing-function-docstring
|
||||
body = self.make_preview(key, mode="html")
|
||||
self.assertIn("Diff warning for ", body)
|
||||
|
|
@ -34,10 +34,6 @@ class TestImportCommandHandler(DataTestCase):
|
|||
handler = self.make_handler(import_handler=myhandler)
|
||||
self.assertIs(handler.import_handler, myhandler)
|
||||
|
||||
# as key
|
||||
handler = self.make_handler(key="import.to_wutta.from_csv")
|
||||
self.assertIsInstance(handler.import_handler, FromCsvToWutta)
|
||||
|
||||
def test_run(self):
|
||||
handler = self.make_handler(
|
||||
import_handler="wuttasync.importing.csv:FromCsvToWutta"
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from unittest import TestCase
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from wuttasync.cli import import_versions as mod, ImportCommandHandler
|
||||
|
||||
|
||||
class TestImportCsv(TestCase):
|
||||
|
||||
def test_basic(self):
|
||||
params = {
|
||||
"models": [],
|
||||
"create": True,
|
||||
"update": True,
|
||||
"delete": False,
|
||||
"dry_run": True,
|
||||
}
|
||||
ctx = MagicMock(params=params)
|
||||
with patch.object(ImportCommandHandler, "run") as run:
|
||||
mod.import_versions(ctx)
|
||||
run.assert_called_once_with(params)
|
||||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
from unittest.mock import patch
|
||||
|
||||
from sqlalchemy import orm
|
||||
|
||||
from wuttjamaican.testing import DataTestCase
|
||||
|
||||
from wuttasync.importing import base as mod, ImportHandler, Orientation
|
||||
|
|
@ -80,31 +78,13 @@ class TestImporter(DataTestCase):
|
|||
|
||||
def test_get_keys(self):
|
||||
model = self.app.model
|
||||
|
||||
# nb. get_keys() will cache the return value, so must
|
||||
# re-create importer for each test
|
||||
|
||||
# keys inspected from model by default
|
||||
imp = self.make_importer(model_class=model.Setting)
|
||||
self.assertEqual(imp.get_keys(), ["name"])
|
||||
imp = self.make_importer(model_class=model.User)
|
||||
self.assertEqual(imp.get_keys(), ["uuid"])
|
||||
|
||||
# class may define 'keys'
|
||||
imp = self.make_importer(model_class=model.User)
|
||||
with patch.object(imp, "keys", new=["foo", "bar"], create=True):
|
||||
with patch.multiple(imp, create=True, key="value"):
|
||||
self.assertEqual(imp.get_keys(), ["value"])
|
||||
with patch.multiple(imp, create=True, keys=["foo", "bar"]):
|
||||
self.assertEqual(imp.get_keys(), ["foo", "bar"])
|
||||
|
||||
# class may define 'key'
|
||||
imp = self.make_importer(model_class=model.User)
|
||||
with patch.object(imp, "key", new="whatever", create=True):
|
||||
self.assertEqual(imp.get_keys(), ["whatever"])
|
||||
|
||||
# class may define 'default_keys'
|
||||
imp = self.make_importer(model_class=model.User)
|
||||
with patch.object(imp, "default_keys", new=["baz", "foo"]):
|
||||
self.assertEqual(imp.get_keys(), ["baz", "foo"])
|
||||
|
||||
def test_process_data(self):
|
||||
model = self.app.model
|
||||
imp = self.make_importer(
|
||||
|
|
@ -671,106 +651,6 @@ class TestFromFile(DataTestCase):
|
|||
close.assert_called_once_with()
|
||||
|
||||
|
||||
class TestQueryWrapper(DataTestCase):
|
||||
|
||||
def test_basic(self):
|
||||
model = self.app.model
|
||||
|
||||
p1 = model.Person(full_name="John Doe")
|
||||
self.session.add(p1)
|
||||
p2 = model.Person(full_name="Jane Doe")
|
||||
self.session.add(p2)
|
||||
self.session.commit()
|
||||
|
||||
# cannot get count via len(query), must use query.count()
|
||||
query = self.session.query(model.Person)
|
||||
self.assertEqual(query.count(), 2)
|
||||
self.assertRaises(TypeError, len, query)
|
||||
|
||||
# but can use len(wrapper)
|
||||
wrapper = mod.QueryWrapper(query)
|
||||
self.assertEqual(len(wrapper), 2)
|
||||
|
||||
# iter(wrapper) should work too
|
||||
people = [p for p in wrapper]
|
||||
self.assertEqual(people, [p1, p2])
|
||||
people = [p for p in iter(wrapper)]
|
||||
self.assertEqual(people, [p1, p2])
|
||||
people = [p for p in list(wrapper)]
|
||||
self.assertEqual(people, [p1, p2])
|
||||
|
||||
|
||||
class TestFromSqlalchemy(DataTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.setup_db()
|
||||
self.handler = ImportHandler(self.config)
|
||||
|
||||
def make_importer(self, **kwargs):
|
||||
kwargs.setdefault("handler", self.handler)
|
||||
return mod.FromSqlalchemy(self.config, **kwargs)
|
||||
|
||||
def test_get_source_query(self):
|
||||
model = self.app.model
|
||||
imp = self.make_importer(
|
||||
source_model_class=model.Upgrade, source_session=self.session
|
||||
)
|
||||
query = imp.get_source_query()
|
||||
self.assertIsInstance(query, orm.Query)
|
||||
froms = query.selectable.get_final_froms()
|
||||
self.assertEqual(len(froms), 1)
|
||||
table = froms[0]
|
||||
self.assertEqual(table.name, "upgrade")
|
||||
|
||||
def test_get_source_objects(self):
|
||||
model = self.app.model
|
||||
|
||||
user1 = model.User(username="fred")
|
||||
self.session.add(user1)
|
||||
user2 = model.User(username="bettie")
|
||||
self.session.add(user2)
|
||||
self.session.commit()
|
||||
|
||||
imp = self.make_importer(
|
||||
source_model_class=model.User, source_session=self.session
|
||||
)
|
||||
result = imp.get_source_objects()
|
||||
self.assertIsInstance(result, mod.QueryWrapper)
|
||||
self.assertEqual(len(result), 2)
|
||||
self.assertEqual(list(result), [user1, user2])
|
||||
|
||||
|
||||
class TestFromSqlalchemyMirror(DataTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.setup_db()
|
||||
self.handler = ImportHandler(self.config)
|
||||
|
||||
def make_importer(self, **kwargs):
|
||||
kwargs.setdefault("handler", self.handler)
|
||||
return mod.FromSqlalchemyMirror(self.config, **kwargs)
|
||||
|
||||
def test_source_model_class(self):
|
||||
model = self.app.model
|
||||
|
||||
# source_model_class will mirror model_class
|
||||
imp = self.make_importer(model_class=model.Upgrade)
|
||||
self.assertIs(imp.model_class, model.Upgrade)
|
||||
self.assertIs(imp.source_model_class, model.Upgrade)
|
||||
|
||||
def test_normalize_source_object(self):
|
||||
model = self.app.model
|
||||
imp = self.make_importer(model_class=model.Upgrade)
|
||||
upgrade = model.Upgrade()
|
||||
|
||||
# normalize_source_object() should invoke normalize_target_object()
|
||||
with patch.object(imp, "normalize_target_object") as normalize_target_object:
|
||||
normalize_target_object.return_value = 42
|
||||
result = imp.normalize_source_object(upgrade)
|
||||
self.assertEqual(result, 42)
|
||||
normalize_target_object.assert_called_once_with(upgrade)
|
||||
|
||||
|
||||
class TestToSqlalchemy(DataTestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
|
|||
|
|
@ -2,18 +2,12 @@
|
|||
|
||||
from collections import OrderedDict
|
||||
from unittest.mock import patch
|
||||
from uuid import UUID
|
||||
|
||||
from wuttjamaican.testing import DataTestCase
|
||||
|
||||
from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy
|
||||
|
||||
|
||||
class FromFooToBar(mod.ImportHandler):
|
||||
source_key = "foo"
|
||||
target_key = "bar"
|
||||
|
||||
|
||||
class TestImportHandler(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
|
|
@ -36,10 +30,10 @@ class TestImportHandler(DataTestCase):
|
|||
|
||||
def test_get_key(self):
|
||||
handler = self.make_handler()
|
||||
self.assertEqual(handler.get_key(), "import.to_None.from_None")
|
||||
self.assertEqual(handler.get_key(), "to_None.from_None.import")
|
||||
|
||||
with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"):
|
||||
self.assertEqual(handler.get_key(), "import.to_wutta.from_csv")
|
||||
self.assertEqual(handler.get_key(), "to_wutta.from_csv.import")
|
||||
|
||||
def test_get_spec(self):
|
||||
handler = self.make_handler()
|
||||
|
|
@ -155,41 +149,15 @@ class TestImportHandler(DataTestCase):
|
|||
kw = {}
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertEqual(result, {})
|
||||
|
||||
# dry_run (not consumed)
|
||||
# captures dry-run flag
|
||||
self.assertFalse(handler.dry_run)
|
||||
kw["dry_run"] = True
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertIn("dry_run", kw)
|
||||
self.assertTrue(kw["dry_run"])
|
||||
self.assertTrue(handler.dry_run)
|
||||
|
||||
# warnings (consumed)
|
||||
self.assertFalse(handler.warnings)
|
||||
kw["warnings"] = True
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertNotIn("warnings", kw)
|
||||
self.assertTrue(handler.warnings)
|
||||
|
||||
# warnings_recipients (consumed)
|
||||
self.assertIsNone(handler.warnings_recipients)
|
||||
kw["warnings_recipients"] = "bob@example.com"
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertNotIn("warnings_recipients", kw)
|
||||
self.assertEqual(handler.warnings_recipients, ["bob@example.com"])
|
||||
|
||||
# warnings_max_diffs (consumed)
|
||||
self.assertEqual(handler.warnings_max_diffs, 15)
|
||||
kw["warnings_max_diffs"] = 30
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertNotIn("warnings_max_diffs", kw)
|
||||
self.assertEqual(handler.warnings_max_diffs, 30)
|
||||
|
||||
def test_define_importers(self):
|
||||
handler = self.make_handler()
|
||||
importers = handler.define_importers()
|
||||
|
|
@ -219,94 +187,6 @@ class TestImportHandler(DataTestCase):
|
|||
KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting
|
||||
)
|
||||
|
||||
def test_get_warnings_email_key(self):
|
||||
handler = FromFooToBar(self.config)
|
||||
|
||||
# default
|
||||
key = handler.get_warnings_email_key()
|
||||
self.assertEqual(key, "import_to_bar_from_foo_warning")
|
||||
|
||||
# override
|
||||
handler.warnings_email_key = "from_foo_to_bar"
|
||||
key = handler.get_warnings_email_key()
|
||||
self.assertEqual(key, "from_foo_to_bar")
|
||||
|
||||
def test_process_changes(self):
|
||||
model = self.app.model
|
||||
handler = self.make_handler()
|
||||
email_handler = self.app.get_email_handler()
|
||||
|
||||
handler.process_started = self.app.localtime()
|
||||
|
||||
alice = model.User(username="alice")
|
||||
bob = model.User(username="bob")
|
||||
charlie = model.User(username="charlie")
|
||||
|
||||
changes = {
|
||||
"User": (
|
||||
[
|
||||
(
|
||||
alice,
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"),
|
||||
"username": "alice",
|
||||
},
|
||||
),
|
||||
],
|
||||
[
|
||||
(
|
||||
bob,
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
||||
"username": "bob",
|
||||
},
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
||||
"username": "bobbie",
|
||||
},
|
||||
),
|
||||
],
|
||||
[
|
||||
(
|
||||
charlie,
|
||||
{
|
||||
"uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"),
|
||||
"username": "charlie",
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
# no email if not in warnings mode
|
||||
self.assertFalse(handler.warnings)
|
||||
with patch.object(self.app, "send_email") as send_email:
|
||||
handler.process_changes(changes)
|
||||
send_email.assert_not_called()
|
||||
|
||||
# email sent (to default recip) if in warnings mode
|
||||
handler.warnings = True
|
||||
self.config.setdefault("wutta.email.default.to", "admin@example.com")
|
||||
with patch.object(email_handler, "deliver_message") as deliver_message:
|
||||
handler.process_changes(changes)
|
||||
deliver_message.assert_called_once()
|
||||
args, kwargs = deliver_message.call_args
|
||||
self.assertEqual(kwargs, {"recips": None})
|
||||
self.assertEqual(len(args), 1)
|
||||
msg = args[0]
|
||||
self.assertEqual(msg.to, ["admin@example.com"])
|
||||
|
||||
# can override email recip
|
||||
handler.warnings_recipients = ["bob@example.com"]
|
||||
with patch.object(email_handler, "deliver_message") as deliver_message:
|
||||
handler.process_changes(changes)
|
||||
deliver_message.assert_called_once()
|
||||
args, kwargs = deliver_message.call_args
|
||||
self.assertEqual(kwargs, {"recips": None})
|
||||
self.assertEqual(len(args), 1)
|
||||
msg = args[0]
|
||||
self.assertEqual(msg.to, ["bob@example.com"])
|
||||
|
||||
|
||||
class TestFromFileHandler(DataTestCase):
|
||||
|
||||
|
|
@ -333,97 +213,6 @@ class TestFromFileHandler(DataTestCase):
|
|||
process_data.assert_called_once_with(input_file_dir=self.tempdir)
|
||||
|
||||
|
||||
class TestFromSqlalchemyHandler(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return mod.FromSqlalchemyHandler(self.config, **kwargs)
|
||||
|
||||
def test_make_source_session(self):
|
||||
handler = self.make_handler()
|
||||
self.assertRaises(NotImplementedError, handler.make_source_session)
|
||||
|
||||
def test_begin_source_transaction(self):
|
||||
handler = self.make_handler()
|
||||
self.assertIsNone(handler.source_session)
|
||||
with patch.object(handler, "make_source_session", return_value=self.session):
|
||||
handler.begin_source_transaction()
|
||||
self.assertIs(handler.source_session, self.session)
|
||||
|
||||
def test_commit_source_transaction(self):
|
||||
model = self.app.model
|
||||
handler = self.make_handler()
|
||||
handler.source_session = self.session
|
||||
self.assertEqual(self.session.query(model.User).count(), 0)
|
||||
|
||||
# nb. do not commit this yet
|
||||
user = model.User(username="fred")
|
||||
self.session.add(user)
|
||||
|
||||
self.assertTrue(self.session.in_transaction())
|
||||
self.assertIn(user, self.session)
|
||||
handler.commit_source_transaction()
|
||||
self.assertIsNone(handler.source_session)
|
||||
self.assertFalse(self.session.in_transaction())
|
||||
self.assertNotIn(user, self.session) # hm, surprising?
|
||||
self.assertEqual(self.session.query(model.User).count(), 1)
|
||||
|
||||
def test_rollback_source_transaction(self):
|
||||
model = self.app.model
|
||||
handler = self.make_handler()
|
||||
handler.source_session = self.session
|
||||
self.assertEqual(self.session.query(model.User).count(), 0)
|
||||
|
||||
# nb. do not commit this yet
|
||||
user = model.User(username="fred")
|
||||
self.session.add(user)
|
||||
|
||||
self.assertTrue(self.session.in_transaction())
|
||||
self.assertIn(user, self.session)
|
||||
handler.rollback_source_transaction()
|
||||
self.assertIsNone(handler.source_session)
|
||||
self.assertFalse(self.session.in_transaction())
|
||||
self.assertNotIn(user, self.session)
|
||||
self.assertEqual(self.session.query(model.User).count(), 0)
|
||||
|
||||
def test_get_importer_kwargs(self):
|
||||
handler = self.make_handler()
|
||||
handler.source_session = self.session
|
||||
kw = handler.get_importer_kwargs("User")
|
||||
self.assertIn("source_session", kw)
|
||||
self.assertIs(kw["source_session"], self.session)
|
||||
|
||||
|
||||
class TestFromWuttaHandler(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return mod.FromWuttaHandler(self.config, **kwargs)
|
||||
|
||||
def test_get_source_title(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
# uses app title by default
|
||||
self.config.setdefault("wutta.app_title", "What About This")
|
||||
self.assertEqual(handler.get_source_title(), "What About This")
|
||||
|
||||
# or generic default if present
|
||||
handler.generic_source_title = "WHATABOUTTHIS"
|
||||
self.assertEqual(handler.get_source_title(), "WHATABOUTTHIS")
|
||||
|
||||
# but prefer specific title if present
|
||||
handler.source_title = "what_about_this"
|
||||
self.assertEqual(handler.get_source_title(), "what_about_this")
|
||||
|
||||
def test_make_source_session(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
# makes "new" (mocked in our case) app session
|
||||
with patch.object(self.app, "make_session") as make_session:
|
||||
make_session.return_value = self.session
|
||||
session = handler.make_source_session()
|
||||
make_session.assert_called_once_with()
|
||||
self.assertIs(session, self.session)
|
||||
|
||||
|
||||
class TestToSqlalchemyHandler(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
|
|
@ -467,34 +256,3 @@ class TestToSqlalchemyHandler(DataTestCase):
|
|||
kw = handler.get_importer_kwargs("Setting")
|
||||
self.assertIn("target_session", kw)
|
||||
self.assertIs(kw["target_session"], self.session)
|
||||
|
||||
|
||||
class TestToWuttaHandler(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return mod.ToWuttaHandler(self.config, **kwargs)
|
||||
|
||||
def test_get_target_title(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
# uses app title by default
|
||||
self.config.setdefault("wutta.app_title", "What About This")
|
||||
self.assertEqual(handler.get_target_title(), "What About This")
|
||||
|
||||
# or generic default if present
|
||||
handler.generic_target_title = "WHATABOUTTHIS"
|
||||
self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS")
|
||||
|
||||
# but prefer specific title if present
|
||||
handler.target_title = "what_about_this"
|
||||
self.assertEqual(handler.get_target_title(), "what_about_this")
|
||||
|
||||
def test_make_target_session(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
# makes "new" (mocked in our case) app session
|
||||
with patch.object(self.app, "make_session") as make_session:
|
||||
make_session.return_value = self.session
|
||||
session = handler.make_target_session()
|
||||
make_session.assert_called_once_with()
|
||||
self.assertIs(session, self.session)
|
||||
|
|
|
|||
|
|
@ -1,262 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from sqlalchemy import orm
|
||||
import sqlalchemy_continuum as continuum
|
||||
|
||||
from wuttjamaican.util import make_true_uuid
|
||||
from wutta_continuum.testing import VersionTestCase
|
||||
|
||||
from wuttasync.importing import versions as mod, Importer
|
||||
|
||||
|
||||
class TestFromWuttaToVersions(VersionTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return mod.FromWuttaToVersions(self.config, **kwargs)
|
||||
|
||||
def test_consume_kwargs(self):
|
||||
|
||||
# no comment by default
|
||||
handler = self.make_handler()
|
||||
kw = handler.consume_kwargs({})
|
||||
self.assertEqual(kw, {})
|
||||
self.assertIsNone(handler.continuum_comment)
|
||||
|
||||
# but can provide one
|
||||
handler = self.make_handler()
|
||||
kw = handler.consume_kwargs({"comment": "yeehaw"})
|
||||
self.assertEqual(kw, {})
|
||||
self.assertEqual(handler.continuum_comment, "yeehaw")
|
||||
|
||||
def test_begin_target_transaction(self):
|
||||
model = self.app.model
|
||||
txncls = continuum.transaction_class(model.User)
|
||||
|
||||
# basic / defaults
|
||||
handler = self.make_handler()
|
||||
self.assertIsNone(handler.continuum_uow)
|
||||
self.assertIsNone(handler.continuum_txn)
|
||||
handler.begin_target_transaction()
|
||||
self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork)
|
||||
self.assertIsInstance(handler.continuum_txn, txncls)
|
||||
# nb. no comment
|
||||
self.assertIsNone(handler.continuum_txn.meta.get("comment"))
|
||||
|
||||
# with comment
|
||||
handler = self.make_handler()
|
||||
handler.continuum_comment = "yeehaw"
|
||||
handler.begin_target_transaction()
|
||||
self.assertIn("comment", handler.continuum_txn.meta)
|
||||
self.assertEqual(handler.continuum_txn.meta["comment"], "yeehaw")
|
||||
|
||||
def test_get_importer_kwargs(self):
|
||||
handler = self.make_handler()
|
||||
handler.begin_target_transaction()
|
||||
|
||||
kw = handler.get_importer_kwargs("User")
|
||||
self.assertIn("continuum_txn", kw)
|
||||
self.assertIs(kw["continuum_txn"], handler.continuum_txn)
|
||||
|
||||
def test_make_importer_factory(self):
|
||||
model = self.app.model
|
||||
handler = self.make_handler()
|
||||
|
||||
# versioned class
|
||||
factory = handler.make_importer_factory(model.User, "User")
|
||||
self.assertTrue(issubclass(factory, mod.FromWuttaToVersionBase))
|
||||
self.assertIs(factory.source_model_class, model.User)
|
||||
self.assertIs(factory.model_class, continuum.version_class(model.User))
|
||||
|
||||
# non-versioned
|
||||
factory = handler.make_importer_factory(model.Upgrade, "Upgrade")
|
||||
self.assertIsNone(factory)
|
||||
|
||||
def test_define_importers(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
importers = handler.define_importers()
|
||||
self.assertIn("User", importers)
|
||||
self.assertIn("Person", importers)
|
||||
self.assertNotIn("Upgrade", importers)
|
||||
|
||||
|
||||
class TestFromWuttaToVersionBase(VersionTestCase):
|
||||
|
||||
def make_importer(self, model_class=None, **kwargs):
|
||||
imp = mod.FromWuttaToVersionBase(self.config, **kwargs)
|
||||
if model_class:
|
||||
imp.model_class = model_class
|
||||
return imp
|
||||
|
||||
def test_get_simple_fields(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
|
||||
# first confirm what a "normal" importer would do
|
||||
imp = Importer(self.config, model_class=vercls)
|
||||
fields = imp.get_simple_fields()
|
||||
self.assertIn("username", fields)
|
||||
self.assertIn("person_uuid", fields)
|
||||
self.assertIn("transaction_id", fields)
|
||||
self.assertIn("operation_type", fields)
|
||||
self.assertIn("end_transaction_id", fields)
|
||||
|
||||
# now test what the "version" importer does
|
||||
imp = self.make_importer(model_class=vercls)
|
||||
fields = imp.get_simple_fields()
|
||||
self.assertIn("username", fields)
|
||||
self.assertIn("person_uuid", fields)
|
||||
self.assertNotIn("transaction_id", fields)
|
||||
self.assertNotIn("operation_type", fields)
|
||||
self.assertNotIn("end_transaction_id", fields)
|
||||
|
||||
def test_get_target_query(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
imp = self.make_importer(model_class=vercls, target_session=self.session)
|
||||
|
||||
# TODO: not sure what else to test here..
|
||||
query = imp.get_target_query()
|
||||
self.assertIsInstance(query, orm.Query)
|
||||
|
||||
def test_normalize_target_object(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
imp = self.make_importer(model_class=vercls)
|
||||
|
||||
user = model.User(username="fred")
|
||||
self.session.add(user)
|
||||
self.session.commit()
|
||||
version = user.versions[0]
|
||||
|
||||
# version object should be embedded in data dict
|
||||
data = imp.normalize_target_object(version)
|
||||
self.assertIsInstance(data, dict)
|
||||
self.assertIn("_objref", data)
|
||||
self.assertIs(data["_objref"], version)
|
||||
|
||||
# but normal object is not embedded
|
||||
data = imp.normalize_target_object(user)
|
||||
self.assertIsInstance(data, dict)
|
||||
self.assertNotIn("_version", data)
|
||||
|
||||
def test_make_version(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
|
||||
user = model.User(username="fred")
|
||||
self.session.add(user)
|
||||
self.session.commit()
|
||||
|
||||
handler = mod.FromWuttaToVersions(self.config)
|
||||
handler.begin_target_transaction()
|
||||
handler.target_session.close()
|
||||
handler.target_session = self.session
|
||||
|
||||
imp = self.make_importer(
|
||||
model_class=vercls,
|
||||
fields=["uuid", "username"],
|
||||
keys=("uuid",),
|
||||
target_session=self.session,
|
||||
continuum_txn=handler.continuum_txn,
|
||||
)
|
||||
|
||||
data = {"uuid": user.uuid, "username": "freddie"}
|
||||
version = imp.make_version(data, continuum.Operation.UPDATE)
|
||||
self.assertIsInstance(version, vercls)
|
||||
self.assertEqual(version.uuid, user.uuid)
|
||||
self.assertEqual(version.username, "freddie")
|
||||
self.assertIn(version, self.session)
|
||||
self.assertIs(version.transaction, imp.continuum_txn)
|
||||
self.assertEqual(version.operation_type, continuum.Operation.UPDATE)
|
||||
|
||||
def test_create_target_object(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
|
||||
handler = mod.FromWuttaToVersions(self.config)
|
||||
handler.begin_target_transaction()
|
||||
handler.target_session.close()
|
||||
handler.target_session = self.session
|
||||
|
||||
imp = self.make_importer(
|
||||
model_class=vercls,
|
||||
fields=["uuid", "username"],
|
||||
keys=("uuid",),
|
||||
target_session=self.session,
|
||||
continuum_txn=handler.continuum_txn,
|
||||
)
|
||||
|
||||
source_data = {"uuid": make_true_uuid(), "username": "bettie"}
|
||||
self.assertEqual(self.session.query(vercls).count(), 0)
|
||||
version = imp.create_target_object((source_data["uuid"], 1), source_data)
|
||||
self.assertEqual(self.session.query(vercls).count(), 1)
|
||||
self.assertEqual(version.transaction_id, imp.continuum_txn.id)
|
||||
self.assertEqual(version.operation_type, continuum.Operation.INSERT)
|
||||
self.assertIsNone(version.end_transaction_id)
|
||||
|
||||
def test_update_target_object(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
|
||||
user = model.User(username="fred")
|
||||
self.session.add(user)
|
||||
self.session.commit()
|
||||
version1 = user.versions[0]
|
||||
|
||||
handler = mod.FromWuttaToVersions(self.config)
|
||||
handler.begin_target_transaction()
|
||||
handler.target_session.close()
|
||||
handler.target_session = self.session
|
||||
|
||||
imp = self.make_importer(
|
||||
model_class=vercls,
|
||||
fields=["uuid", "username"],
|
||||
keys=("uuid",),
|
||||
target_session=self.session,
|
||||
continuum_txn=handler.continuum_txn,
|
||||
)
|
||||
|
||||
source_data = {"uuid": user.uuid, "username": "freddie"}
|
||||
target_data = imp.normalize_target_object(version1)
|
||||
self.assertEqual(self.session.query(vercls).count(), 1)
|
||||
self.assertIsNone(version1.end_transaction_id)
|
||||
version2 = imp.update_target_object(
|
||||
version1, source_data, target_data=target_data
|
||||
)
|
||||
self.assertEqual(self.session.query(vercls).count(), 2)
|
||||
self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id)
|
||||
self.assertEqual(version2.transaction_id, imp.continuum_txn.id)
|
||||
self.assertEqual(version2.operation_type, continuum.Operation.UPDATE)
|
||||
self.assertIsNone(version2.end_transaction_id)
|
||||
|
||||
def test_delete_target_object(self):
|
||||
model = self.app.model
|
||||
vercls = continuum.version_class(model.User)
|
||||
|
||||
user = model.User(username="fred")
|
||||
self.session.add(user)
|
||||
self.session.commit()
|
||||
version1 = user.versions[0]
|
||||
|
||||
handler = mod.FromWuttaToVersions(self.config)
|
||||
handler.begin_target_transaction()
|
||||
handler.target_session.close()
|
||||
handler.target_session = self.session
|
||||
|
||||
imp = self.make_importer(
|
||||
model_class=vercls,
|
||||
fields=["uuid", "username"],
|
||||
keys=("uuid",),
|
||||
target_session=self.session,
|
||||
continuum_txn=handler.continuum_txn,
|
||||
)
|
||||
|
||||
self.assertEqual(self.session.query(vercls).count(), 1)
|
||||
self.assertIsNone(version1.end_transaction_id)
|
||||
version2 = imp.delete_target_object(version1)
|
||||
self.assertEqual(self.session.query(vercls).count(), 2)
|
||||
self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id)
|
||||
self.assertEqual(version2.transaction_id, imp.continuum_txn.id)
|
||||
self.assertEqual(version2.operation_type, continuum.Operation.DELETE)
|
||||
self.assertIsNone(version2.end_transaction_id)
|
||||
|
|
@ -1,3 +1,38 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from wuttjamaican.testing import DataTestCase
|
||||
|
||||
from wuttasync.importing import wutta as mod
|
||||
|
||||
|
||||
class TestToWuttaHandler(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return mod.ToWuttaHandler(self.config, **kwargs)
|
||||
|
||||
def test_get_target_title(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
# uses app title by default
|
||||
self.config.setdefault("wutta.app_title", "What About This")
|
||||
self.assertEqual(handler.get_target_title(), "What About This")
|
||||
|
||||
# or generic default if present
|
||||
handler.generic_target_title = "WHATABOUTTHIS"
|
||||
self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS")
|
||||
|
||||
# but prefer specific title if present
|
||||
handler.target_title = "what_about_this"
|
||||
self.assertEqual(handler.get_target_title(), "what_about_this")
|
||||
|
||||
def test_make_target_session(self):
|
||||
handler = self.make_handler()
|
||||
|
||||
# makes "new" (mocked in our case) app session
|
||||
with patch.object(self.app, "make_session") as make_session:
|
||||
make_session.return_value = self.session
|
||||
session = handler.make_target_session()
|
||||
make_session.assert_called_once_with()
|
||||
self.assertIs(session, self.session)
|
||||
|
|
|
|||
|
|
@ -1,126 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from wuttjamaican.testing import ConfigTestCase
|
||||
|
||||
from wuttasync import app as mod
|
||||
from wuttasync.importing import ImportHandler
|
||||
from wuttasync.importing.csv import FromCsvToWutta
|
||||
|
||||
|
||||
class FromFooToBar(ImportHandler):
|
||||
source_key = "foo"
|
||||
target_key = "bar"
|
||||
|
||||
|
||||
class FromCsvToPoser(FromCsvToWutta):
|
||||
pass
|
||||
|
||||
|
||||
class TestWuttaSyncAppProvider(ConfigTestCase):
|
||||
|
||||
def test_get_all_import_handlers(self):
|
||||
|
||||
# by default our custom handler is not found
|
||||
handlers = self.app.get_all_import_handlers()
|
||||
self.assertIn(FromCsvToWutta, handlers)
|
||||
self.assertNotIn(FromFooToBar, handlers)
|
||||
|
||||
# make sure if we configure a custom handler, it is found
|
||||
self.config.setdefault(
|
||||
"wuttasync.importing.import.to_wutta.from_csv.handler",
|
||||
"tests.test_app:FromFooToBar",
|
||||
)
|
||||
handlers = self.app.get_all_import_handlers()
|
||||
self.assertIn(FromCsvToWutta, handlers)
|
||||
self.assertIn(FromFooToBar, handlers)
|
||||
|
||||
def test_get_designated_import_handler_spec(self):
|
||||
|
||||
# fetch of unknown key returns none
|
||||
spec = self.app.get_designated_import_handler_spec("test01")
|
||||
self.assertIsNone(spec)
|
||||
|
||||
# unless we require it, in which case, error
|
||||
self.assertRaises(
|
||||
ValueError,
|
||||
self.app.get_designated_import_handler_spec,
|
||||
"test01",
|
||||
require=True,
|
||||
)
|
||||
|
||||
# we configure one for whatever key we like
|
||||
self.config.setdefault(
|
||||
"wuttasync.importing.test02.handler", "tests.test_app:FromBarToFoo"
|
||||
)
|
||||
spec = self.app.get_designated_import_handler_spec("test02")
|
||||
self.assertEqual(spec, "tests.test_app:FromBarToFoo")
|
||||
|
||||
# we can also define a "default" designated handler
|
||||
self.config.setdefault(
|
||||
"wuttasync.importing.test03.default_handler",
|
||||
"tests.test_app:FromBarToFoo",
|
||||
)
|
||||
spec = self.app.get_designated_import_handler_spec("test03")
|
||||
self.assertEqual(spec, "tests.test_app:FromBarToFoo")
|
||||
|
||||
def test_get_designated_import_handlers(self):
|
||||
|
||||
# some designated handlers exist, but not our custom handler
|
||||
handlers = self.app.get_designated_import_handlers()
|
||||
csv_handlers = [
|
||||
h for h in handlers if h.get_key() == "import.to_wutta.from_csv"
|
||||
]
|
||||
self.assertEqual(len(csv_handlers), 1)
|
||||
csv_handler = csv_handlers[0]
|
||||
self.assertIsInstance(csv_handler, FromCsvToWutta)
|
||||
self.assertFalse(isinstance(csv_handler, FromCsvToPoser))
|
||||
self.assertFalse(
|
||||
any([h.get_key() == "import.to_bar.from_foo" for h in handlers])
|
||||
)
|
||||
self.assertFalse(any([isinstance(h, FromFooToBar) for h in handlers]))
|
||||
self.assertFalse(any([isinstance(h, FromCsvToPoser) for h in handlers]))
|
||||
self.assertTrue(
|
||||
any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
|
||||
)
|
||||
|
||||
# but we can make custom designated
|
||||
self.config.setdefault(
|
||||
"wuttasync.importing.import.to_wutta.from_csv.handler",
|
||||
"tests.test_app:FromCsvToPoser",
|
||||
)
|
||||
handlers = self.app.get_designated_import_handlers()
|
||||
csv_handlers = [
|
||||
h for h in handlers if h.get_key() == "import.to_wutta.from_csv"
|
||||
]
|
||||
self.assertEqual(len(csv_handlers), 1)
|
||||
csv_handler = csv_handlers[0]
|
||||
self.assertIsInstance(csv_handler, FromCsvToWutta)
|
||||
self.assertIsInstance(csv_handler, FromCsvToPoser)
|
||||
self.assertTrue(
|
||||
any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
|
||||
)
|
||||
|
||||
def test_get_import_handler(self):
|
||||
|
||||
# make sure a basic fetch works
|
||||
handler = self.app.get_import_handler("import.to_wutta.from_csv")
|
||||
self.assertIsInstance(handler, FromCsvToWutta)
|
||||
self.assertFalse(isinstance(handler, FromCsvToPoser))
|
||||
|
||||
# and make sure custom override works
|
||||
self.config.setdefault(
|
||||
"wuttasync.importing.import.to_wutta.from_csv.handler",
|
||||
"tests.test_app:FromCsvToPoser",
|
||||
)
|
||||
handler = self.app.get_import_handler("import.to_wutta.from_csv")
|
||||
self.assertIsInstance(handler, FromCsvToWutta)
|
||||
self.assertIsInstance(handler, FromCsvToPoser)
|
||||
|
||||
# unknown importer cannot be found
|
||||
handler = self.app.get_import_handler("bogus")
|
||||
self.assertIsNone(handler)
|
||||
|
||||
# and if we require it, error will raise
|
||||
self.assertRaises(
|
||||
ValueError, self.app.get_import_handler, "bogus", require=True
|
||||
)
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from wuttjamaican.testing import ConfigTestCase
|
||||
|
||||
from wuttasync import emails as mod
|
||||
from wuttasync.importing import ImportHandler
|
||||
from wuttasync.testing import ImportExportWarningTestCase
|
||||
|
||||
|
||||
class FromFooToWutta(ImportHandler):
|
||||
pass
|
||||
|
||||
|
||||
class TestImportExportWarning(ConfigTestCase):
|
||||
|
||||
def make_setting(self, factory=None):
|
||||
if not factory:
|
||||
factory = mod.ImportExportWarning
|
||||
setting = factory(self.config)
|
||||
return setting
|
||||
|
||||
def test_get_description(self):
|
||||
self.config.setdefault("wutta.app_title", "Wutta Poser")
|
||||
setting = self.make_setting()
|
||||
setting.import_handler_key = "import.to_wutta.from_csv"
|
||||
self.assertEqual(
|
||||
setting.get_description(),
|
||||
"Diff warning email for importing CSV → Wutta Poser",
|
||||
)
|
||||
|
||||
def test_get_default_subject(self):
|
||||
self.config.setdefault("wutta.app_title", "Wutta Poser")
|
||||
setting = self.make_setting()
|
||||
setting.import_handler_key = "import.to_wutta.from_csv"
|
||||
self.assertEqual(setting.get_default_subject(), "Changes for CSV → Wutta Poser")
|
||||
|
||||
def test_get_import_handler(self):
|
||||
|
||||
# nb. typical name pattern
|
||||
class import_to_wutta_from_foo_warning(mod.ImportExportWarning):
|
||||
pass
|
||||
|
||||
# nb. name does not match spec pattern
|
||||
class import_to_wutta_from_bar_blah(mod.ImportExportWarning):
|
||||
pass
|
||||
|
||||
# register our import handler
|
||||
self.config.setdefault(
|
||||
"wuttasync.importing.import.to_wutta.from_foo.handler",
|
||||
"tests.test_emails:FromFooToWutta",
|
||||
)
|
||||
|
||||
# error if spec/key not discoverable
|
||||
setting = self.make_setting(import_to_wutta_from_bar_blah)
|
||||
self.assertRaises(ValueError, setting.get_import_handler)
|
||||
|
||||
# can lookup by name (auto-spec)
|
||||
setting = self.make_setting(import_to_wutta_from_foo_warning)
|
||||
handler = setting.get_import_handler()
|
||||
self.assertIsInstance(handler, FromFooToWutta)
|
||||
|
||||
# can lookup by explicit spec
|
||||
setting = self.make_setting(import_to_wutta_from_bar_blah)
|
||||
setting.import_handler_spec = "tests.test_emails:FromFooToWutta"
|
||||
handler = setting.get_import_handler()
|
||||
self.assertIsInstance(handler, FromFooToWutta)
|
||||
|
||||
# can lookup by explicit key
|
||||
setting = self.make_setting(import_to_wutta_from_bar_blah)
|
||||
setting.import_handler_key = "import.to_wutta.from_foo"
|
||||
handler = setting.get_import_handler()
|
||||
self.assertIsInstance(handler, FromFooToWutta)
|
||||
|
||||
|
||||
class TestEmailSettings(ImportExportWarningTestCase):
|
||||
|
||||
def test_import_to_versions_from_wutta_warning(self):
|
||||
self.do_test_preview("import_to_versions_from_wutta_warning")
|
||||
|
||||
def test_import_to_wutta_from_csv_warning(self):
|
||||
self.do_test_preview("import_to_wutta_from_csv_warning")
|
||||
Loading…
Add table
Add a link
Reference in a new issue