Compare commits
No commits in common. "8c5918b9fb9572b096c60e989aa562b361f8c3b5" and "c38cd2c1792d2a90adbdb6112c81afe2cf1f8bff" have entirely different histories.
8c5918b9fb
...
c38cd2c179
33 changed files with 116 additions and 2588 deletions
19
CHANGELOG.md
19
CHANGELOG.md
|
|
@ -5,25 +5,6 @@ All notable changes to WuttaSync will be documented in this file.
|
||||||
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
||||||
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
## v0.3.0 (2025-12-20)
|
|
||||||
|
|
||||||
### Feat
|
|
||||||
|
|
||||||
- add `warnings` mode for import/export handlers, commands
|
|
||||||
- add the `import-versions` command, handler logic
|
|
||||||
|
|
||||||
### Fix
|
|
||||||
|
|
||||||
- run all models when none specified, for import/export commands
|
|
||||||
- allow passing just `key` to ImportCommandHandler
|
|
||||||
- add `--comment` param for `import-versions` command
|
|
||||||
- add basic data type coercion for CSV -> SQLAlchemy import
|
|
||||||
- refactor some more for tests + pylint
|
|
||||||
- refactor per pylint; add to tox
|
|
||||||
- format all code with black
|
|
||||||
- tweak logging when deleting object
|
|
||||||
- add logging when deleting target object
|
|
||||||
|
|
||||||
## v0.2.1 (2025-06-29)
|
## v0.2.1 (2025-06-29)
|
||||||
|
|
||||||
### Fix
|
### Fix
|
||||||
|
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
|
|
||||||
``wuttasync.app``
|
|
||||||
=================
|
|
||||||
|
|
||||||
.. automodule:: wuttasync.app
|
|
||||||
:members:
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
|
|
||||||
``wuttasync.cli.import_versions``
|
|
||||||
=================================
|
|
||||||
|
|
||||||
.. automodule:: wuttasync.cli.import_versions
|
|
||||||
:members:
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
|
|
||||||
``wuttasync.emails``
|
|
||||||
====================
|
|
||||||
|
|
||||||
.. automodule:: wuttasync.emails
|
|
||||||
:members:
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
|
|
||||||
``wuttasync.importing.versions``
|
|
||||||
================================
|
|
||||||
|
|
||||||
.. automodule:: wuttasync.importing.versions
|
|
||||||
:members:
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
|
|
||||||
``wuttasync.testing``
|
|
||||||
=====================
|
|
||||||
|
|
||||||
.. automodule:: wuttasync.testing
|
|
||||||
:members:
|
|
||||||
|
|
@ -31,13 +31,6 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
|
||||||
intersphinx_mapping = {
|
intersphinx_mapping = {
|
||||||
"python": ("https://docs.python.org/3/", None),
|
"python": ("https://docs.python.org/3/", None),
|
||||||
"rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None),
|
"rattail-manual": ("https://docs.wuttaproject.org/rattail-manual/", None),
|
||||||
"sqlalchemy": ("http://docs.sqlalchemy.org/en/latest/", None),
|
|
||||||
"sqlalchemy-continuum": (
|
|
||||||
"https://sqlalchemy-continuum.readthedocs.io/en/latest/",
|
|
||||||
None,
|
|
||||||
),
|
|
||||||
"sqlalchemy-utils": ("https://sqlalchemy-utils.readthedocs.io/en/latest/", None),
|
|
||||||
"wutta-continuum": ("https://docs.wuttaproject.org/wutta-continuum/", None),
|
|
||||||
"wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None),
|
"wuttjamaican": ("https://docs.wuttaproject.org/wuttjamaican/", None),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,26 +6,6 @@ Glossary
|
||||||
.. glossary::
|
.. glossary::
|
||||||
:sorted:
|
:sorted:
|
||||||
|
|
||||||
import/export key
|
|
||||||
Unique key representing a particular type of import/export job,
|
|
||||||
i.e. the source/target and orientation (import vs. export).
|
|
||||||
|
|
||||||
For instance "Wutta → CSV export" uses the key:
|
|
||||||
``export.to_csv.from_wutta``
|
|
||||||
|
|
||||||
More than one :term:`import handler` can share a key, e.g. one
|
|
||||||
may subclass another and inherit the key.
|
|
||||||
|
|
||||||
However only one handler is "designated" for a given key; it will
|
|
||||||
be used by default for running those jobs.
|
|
||||||
|
|
||||||
This key is used for lookup in
|
|
||||||
:meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`.
|
|
||||||
|
|
||||||
See also
|
|
||||||
:meth:`~wuttasync.importing.handlers.ImportHandler.get_key()`
|
|
||||||
method on the import/export handler.
|
|
||||||
|
|
||||||
import handler
|
import handler
|
||||||
This a type of :term:`handler` which is responsible for a
|
This a type of :term:`handler` which is responsible for a
|
||||||
particular set of data import/export task(s).
|
particular set of data import/export task(s).
|
||||||
|
|
|
||||||
|
|
@ -67,21 +67,16 @@ cf. :doc:`rattail-manual:data/sync/index`.
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 1
|
:maxdepth: 1
|
||||||
:caption: Package API
|
:caption: API
|
||||||
|
|
||||||
api/wuttasync
|
api/wuttasync
|
||||||
api/wuttasync.app
|
|
||||||
api/wuttasync.cli
|
api/wuttasync.cli
|
||||||
api/wuttasync.cli.base
|
api/wuttasync.cli.base
|
||||||
api/wuttasync.cli.import_csv
|
api/wuttasync.cli.import_csv
|
||||||
api/wuttasync.cli.import_versions
|
|
||||||
api/wuttasync.emails
|
|
||||||
api/wuttasync.importing
|
api/wuttasync.importing
|
||||||
api/wuttasync.importing.base
|
api/wuttasync.importing.base
|
||||||
api/wuttasync.importing.csv
|
api/wuttasync.importing.csv
|
||||||
api/wuttasync.importing.handlers
|
api/wuttasync.importing.handlers
|
||||||
api/wuttasync.importing.model
|
api/wuttasync.importing.model
|
||||||
api/wuttasync.importing.versions
|
|
||||||
api/wuttasync.importing.wutta
|
api/wuttasync.importing.wutta
|
||||||
api/wuttasync.testing
|
|
||||||
api/wuttasync.util
|
api/wuttasync.util
|
||||||
|
|
|
||||||
|
|
@ -25,24 +25,3 @@ types may not behave as expected etc.
|
||||||
Defined in: :mod:`wuttasync.cli.import_csv`
|
Defined in: :mod:`wuttasync.cli.import_csv`
|
||||||
|
|
||||||
.. program-output:: wutta import-csv --help
|
.. program-output:: wutta import-csv --help
|
||||||
|
|
||||||
|
|
||||||
.. _wutta-import-versions:
|
|
||||||
|
|
||||||
``wutta import-versions``
|
|
||||||
-------------------------
|
|
||||||
|
|
||||||
Import latest data to version tables, for the Wutta :term:`app
|
|
||||||
database`.
|
|
||||||
|
|
||||||
The purpose of this is to ensure version tables accurately reflect
|
|
||||||
the current "live" data set, for given table(s). It is only
|
|
||||||
relevant/usable if versioning is configured and enabled. For more
|
|
||||||
on that see :doc:`wutta-continuum:index`.
|
|
||||||
|
|
||||||
This command can check/update version tables for any versioned class
|
|
||||||
in the :term:`app model`.
|
|
||||||
|
|
||||||
Defined in: :mod:`wuttasync.cli.import_versions`
|
|
||||||
|
|
||||||
.. program-output:: wutta import-versions --help
|
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "WuttaSync"
|
name = "WuttaSync"
|
||||||
version = "0.3.0"
|
version = "0.2.1"
|
||||||
description = "Wutta Framework for data import/export and real-time sync"
|
description = "Wutta Framework for data import/export and real-time sync"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]
|
authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]
|
||||||
|
|
@ -26,25 +26,17 @@ classifiers = [
|
||||||
]
|
]
|
||||||
requires-python = ">= 3.8"
|
requires-python = ">= 3.8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"humanize",
|
|
||||||
"makefun",
|
"makefun",
|
||||||
"SQLAlchemy-Utils",
|
"SQLAlchemy-Utils",
|
||||||
"WuttJamaican[db]>=0.27.0",
|
"WuttJamaican[db]>=0.16.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
|
docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
|
||||||
tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"]
|
tests = ["pylint", "pytest", "pytest-cov", "tox"]
|
||||||
|
|
||||||
|
|
||||||
[project.entry-points."wutta.app.providers"]
|
|
||||||
wuttasync = "wuttasync.app:WuttaSyncAppProvider"
|
|
||||||
|
|
||||||
[project.entry-points."wuttasync.importing"]
|
|
||||||
"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions"
|
|
||||||
"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta"
|
|
||||||
|
|
||||||
[project.entry-points."wutta.typer_imports"]
|
[project.entry-points."wutta.typer_imports"]
|
||||||
wuttasync = "wuttasync.cli"
|
wuttasync = "wuttasync.cli"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,225 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
################################################################################
|
|
||||||
#
|
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
|
||||||
#
|
|
||||||
# This file is part of Wutta Framework.
|
|
||||||
#
|
|
||||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
|
||||||
# under the terms of the GNU General Public License as published by the Free
|
|
||||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
|
||||||
# later version.
|
|
||||||
#
|
|
||||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
|
||||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
# more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License along with
|
|
||||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
################################################################################
|
|
||||||
"""
|
|
||||||
App handler supplement for WuttaSync
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
from wuttjamaican.app import AppProvider
|
|
||||||
from wuttjamaican.util import load_entry_points
|
|
||||||
|
|
||||||
|
|
||||||
class WuttaSyncAppProvider(AppProvider):
|
|
||||||
"""
|
|
||||||
The :term:`app provider` for WuttaSync.
|
|
||||||
|
|
||||||
This adds some methods to the :term:`app handler`, which are
|
|
||||||
specific to import/export.
|
|
||||||
|
|
||||||
It also declares some :term:`email modules <email module>` and
|
|
||||||
:term:`email templates <email template>` for the app.
|
|
||||||
|
|
||||||
We have two concerns when doing lookups etc. for import/export
|
|
||||||
handlers:
|
|
||||||
|
|
||||||
* which handlers are *available* - i.e. they exist and are
|
|
||||||
discoverable
|
|
||||||
* which handlers are *designated* - only one designated handler
|
|
||||||
per key
|
|
||||||
|
|
||||||
All "available" handlers will have a key, but some keys may be
|
|
||||||
referenced by multiple handlers. For each key, only one handler
|
|
||||||
can be "designated" - there is a default, but config can override.
|
|
||||||
"""
|
|
||||||
|
|
||||||
email_modules = ["wuttasync.emails"]
|
|
||||||
email_templates = ["wuttasync:email-templates"]
|
|
||||||
|
|
||||||
def get_all_import_handlers(self):
|
|
||||||
"""
|
|
||||||
Returns *all* :term:`import/export handler <import handler>`
|
|
||||||
*classes* which are known to exist, i.e. are discoverable.
|
|
||||||
|
|
||||||
See also :meth:`get_import_handler()` and
|
|
||||||
:meth:`get_designated_import_handlers()`.
|
|
||||||
|
|
||||||
The discovery process is as follows:
|
|
||||||
|
|
||||||
* load handlers from registered entry points
|
|
||||||
* check config for designated handlers
|
|
||||||
|
|
||||||
Checking for designated handler config is not a reliable way
|
|
||||||
to discover handlers, but it's done just in case any new ones
|
|
||||||
might be found.
|
|
||||||
|
|
||||||
Registration via entry points is the only way to ensure a
|
|
||||||
handler is discoverable. The entry point group name is always
|
|
||||||
``wuttasync.importing`` regardless of :term:`app name`;
|
|
||||||
entries are like ``"handler_key" = "handler_spec"``. For
|
|
||||||
example:
|
|
||||||
|
|
||||||
.. code-block:: toml
|
|
||||||
|
|
||||||
[project.entry-points."wuttasync.importing"]
|
|
||||||
"export.to_csv.from_poser" = "poser.exporting.csv:FromPoserToCsv"
|
|
||||||
"import.to_poser.from_csv" = "poser.importing.csv:FromCsvToPoser"
|
|
||||||
|
|
||||||
:returns: List of all import/export handler classes
|
|
||||||
"""
|
|
||||||
# first load all "registered" Handler classes
|
|
||||||
factories = load_entry_points("wuttasync.importing", ignore_errors=True)
|
|
||||||
|
|
||||||
# organize registered classes by spec
|
|
||||||
specs = {factory.get_spec(): factory for factory in factories.values()}
|
|
||||||
|
|
||||||
# many handlers may not be registered per se, but may be
|
|
||||||
# designated via config. so try to include those too
|
|
||||||
for factory in factories.values():
|
|
||||||
spec = self.get_designated_import_handler_spec(factory.get_key())
|
|
||||||
if spec and spec not in specs:
|
|
||||||
specs[spec] = self.app.load_object(spec)
|
|
||||||
|
|
||||||
# flatten back to simple list of classes
|
|
||||||
factories = list(specs.values())
|
|
||||||
return factories
|
|
||||||
|
|
||||||
def get_designated_import_handler_spec(self, key, require=False):
|
|
||||||
"""
|
|
||||||
Returns the designated import/export handler :term:`spec`
|
|
||||||
string for the given type key.
|
|
||||||
|
|
||||||
This just checks config for the designated handler, using the
|
|
||||||
``wuttasync.importing`` prefix regardless of :term:`app name`.
|
|
||||||
For instance:
|
|
||||||
|
|
||||||
.. code-block:: ini
|
|
||||||
|
|
||||||
[wuttasync.importing]
|
|
||||||
export.to_csv.from_poser.handler = poser.exporting.csv:FromPoserToCsv
|
|
||||||
import.to_poser.from_csv.handler = poser.importing.csv:FromCsvToPoser
|
|
||||||
|
|
||||||
See also :meth:`get_designated_import_handlers()` and
|
|
||||||
:meth:`get_import_handler()`.
|
|
||||||
|
|
||||||
:param key: Unique key indicating the type of import/export
|
|
||||||
handler.
|
|
||||||
|
|
||||||
:param require: Flag indicating whether an error should be raised if no
|
|
||||||
handler is found.
|
|
||||||
|
|
||||||
:returns: Spec string for the designated handler. If none is
|
|
||||||
configured, then ``None`` is returned *unless* the
|
|
||||||
``require`` param is true, in which case an error is
|
|
||||||
raised.
|
|
||||||
"""
|
|
||||||
spec = self.config.get(f"wuttasync.importing.{key}.handler")
|
|
||||||
if spec:
|
|
||||||
return spec
|
|
||||||
|
|
||||||
spec = self.config.get(f"wuttasync.importing.{key}.default_handler")
|
|
||||||
if spec:
|
|
||||||
return spec
|
|
||||||
|
|
||||||
if require:
|
|
||||||
raise ValueError(f"Cannot locate import handler spec for key: {key}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def get_designated_import_handlers(self):
|
|
||||||
"""
|
|
||||||
Returns all *designated* import/export handler *instances*.
|
|
||||||
|
|
||||||
Each import/export handler has a "key" which indicates the
|
|
||||||
"type" of import/export job it performs. For instance the CSV
|
|
||||||
→ Wutta import has the key: ``import.to_wutta.from_csv``
|
|
||||||
|
|
||||||
More than one handler can be defined for that key; however
|
|
||||||
only one such handler will be "designated" for each key.
|
|
||||||
|
|
||||||
This method first loads *all* available import handlers, then
|
|
||||||
organizes them by key, and tries to determine which handler
|
|
||||||
should be designated for each key.
|
|
||||||
|
|
||||||
See also :meth:`get_all_import_handlers()` and
|
|
||||||
:meth:`get_designated_import_handler_spec()`.
|
|
||||||
|
|
||||||
:returns: List of designated import/export handler instances
|
|
||||||
"""
|
|
||||||
grouped = OrderedDict()
|
|
||||||
for factory in self.get_all_import_handlers():
|
|
||||||
key = factory.get_key()
|
|
||||||
grouped.setdefault(key, []).append(factory)
|
|
||||||
|
|
||||||
def find_designated(key, group):
|
|
||||||
spec = self.get_designated_import_handler_spec(key)
|
|
||||||
if spec:
|
|
||||||
for factory in group:
|
|
||||||
if factory.get_spec() == spec:
|
|
||||||
return factory
|
|
||||||
if len(group) == 1:
|
|
||||||
return group[0]
|
|
||||||
return None
|
|
||||||
|
|
||||||
designated = []
|
|
||||||
for key, group in grouped.items():
|
|
||||||
factory = find_designated(key, group)
|
|
||||||
if factory:
|
|
||||||
handler = factory(self.config)
|
|
||||||
designated.append(handler)
|
|
||||||
|
|
||||||
return designated
|
|
||||||
|
|
||||||
def get_import_handler(self, key, require=False, **kwargs):
|
|
||||||
"""
|
|
||||||
Returns the designated :term:`import/export handler <import
|
|
||||||
handler>` instance for the given :term:`import/export key`.
|
|
||||||
|
|
||||||
See also :meth:`get_all_import_handlers()` and
|
|
||||||
:meth:`get_designated_import_handlers()`.
|
|
||||||
|
|
||||||
:param key: Key indicating the type of import/export handler,
|
|
||||||
e.g. ``"import.to_wutta.from_csv"``
|
|
||||||
|
|
||||||
:param require: Set this to true if you want an error raised
|
|
||||||
when no handler is found.
|
|
||||||
|
|
||||||
:returns: The import/export handler instance. If no handler
|
|
||||||
is found, then ``None`` is returned, unless ``require``
|
|
||||||
param is true, in which case error is raised.
|
|
||||||
"""
|
|
||||||
# first try to fetch the handler per designated spec
|
|
||||||
spec = self.get_designated_import_handler_spec(key, **kwargs)
|
|
||||||
if spec:
|
|
||||||
factory = self.app.load_object(spec)
|
|
||||||
return factory(self.config)
|
|
||||||
|
|
||||||
# nothing was designated, so leverage logic which already
|
|
||||||
# sorts out which handler is "designated" for given key
|
|
||||||
designated = self.get_designated_import_handlers()
|
|
||||||
for handler in designated:
|
|
||||||
if handler.get_key() == key:
|
|
||||||
return handler
|
|
||||||
|
|
||||||
if require:
|
|
||||||
raise ValueError(f"Cannot locate import handler for key: {key}")
|
|
||||||
return None
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
################################################################################
|
################################################################################
|
||||||
#
|
#
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
# Copyright © 2024 Lance Edgar
|
||||||
#
|
#
|
||||||
# This file is part of Wutta Framework.
|
# This file is part of Wutta Framework.
|
||||||
#
|
#
|
||||||
|
|
@ -34,4 +34,3 @@ from .base import import_command, file_import_command, ImportCommandHandler
|
||||||
|
|
||||||
# nb. must bring in all modules for discovery to work
|
# nb. must bring in all modules for discovery to work
|
||||||
from . import import_csv
|
from . import import_csv
|
||||||
from . import import_versions
|
|
||||||
|
|
|
||||||
|
|
@ -50,35 +50,19 @@ class ImportCommandHandler(GenericHandler):
|
||||||
create this handler and call its :meth:`run()` method.
|
create this handler and call its :meth:`run()` method.
|
||||||
|
|
||||||
This handler does not know how to import/export data, but it knows
|
This handler does not know how to import/export data, but it knows
|
||||||
how to make its :attr:`import_handler` do it. Likewise, the
|
how to make its :attr:`import_handler` do it.
|
||||||
import handler is not "CLI-aware" - so this provides the glue.
|
|
||||||
|
|
||||||
:param import_handler: During construction, caller can specify the
|
:param import_handler: During construction, caller can specify the
|
||||||
:attr:`import_handler` as any of:
|
:attr:`import_handler` as any of:
|
||||||
|
|
||||||
* import handler instance
|
* import handler instance
|
||||||
* import handler factory (e.g. class)
|
* import handler factory (e.g. class)
|
||||||
* import handler :term:`spec`
|
* import handler spec (cf. :func:`~wuttjamaican:wuttjamaican.util.load_object()`)
|
||||||
|
|
||||||
:param key: Optional :term:`import/export key` to use for handler
|
For example::
|
||||||
lookup. Only used if ``import_handler`` param is not set.
|
|
||||||
|
|
||||||
Typical usage for custom commands will be to provide the spec::
|
handler = ImportCommandHandler(
|
||||||
|
config, import_handler='wuttasync.importing.csv:FromCsvToWutta')
|
||||||
handler = ImportCommandHandler(
|
|
||||||
config, "poser.importing.foo:FromFooToPoser"
|
|
||||||
)
|
|
||||||
|
|
||||||
Library authors may prefer to use the import/export key; this lets
|
|
||||||
the command work with any designated handler::
|
|
||||||
|
|
||||||
handler = ImportCommandHandler(
|
|
||||||
config, key="import.to_poser.from_foo"
|
|
||||||
)
|
|
||||||
|
|
||||||
See also
|
|
||||||
:meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`
|
|
||||||
which does the lookup by key.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import_handler = None
|
import_handler = None
|
||||||
|
|
@ -87,7 +71,7 @@ class ImportCommandHandler(GenericHandler):
|
||||||
invoked when command runs. See also :meth:`run()`.
|
invoked when command runs. See also :meth:`run()`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, config, import_handler=None, key=None):
|
def __init__(self, config, import_handler=None):
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
|
|
||||||
if import_handler:
|
if import_handler:
|
||||||
|
|
@ -99,9 +83,6 @@ class ImportCommandHandler(GenericHandler):
|
||||||
factory = self.app.load_object(import_handler)
|
factory = self.app.load_object(import_handler)
|
||||||
self.import_handler = factory(self.config)
|
self.import_handler = factory(self.config)
|
||||||
|
|
||||||
elif key:
|
|
||||||
self.import_handler = self.app.get_import_handler(key, require=True)
|
|
||||||
|
|
||||||
def run(self, params, progress=None): # pylint: disable=unused-argument
|
def run(self, params, progress=None): # pylint: disable=unused-argument
|
||||||
"""
|
"""
|
||||||
Run the import/export job(s) based on command line params.
|
Run the import/export job(s) based on command line params.
|
||||||
|
|
@ -125,15 +106,15 @@ class ImportCommandHandler(GenericHandler):
|
||||||
return
|
return
|
||||||
|
|
||||||
# otherwise process some data
|
# otherwise process some data
|
||||||
log.debug("using handler: %s", self.import_handler.get_spec())
|
|
||||||
kw = dict(params)
|
kw = dict(params)
|
||||||
models = kw.pop("models")
|
models = kw.pop("models")
|
||||||
if not models:
|
log.debug("using handler: %s", self.import_handler.get_spec())
|
||||||
models = list(self.import_handler.importers)
|
# TODO: need to use all/default models if none specified
|
||||||
|
# (and should know models by now for logging purposes)
|
||||||
log.debug(
|
log.debug(
|
||||||
"%s %s for models: %s",
|
"running %s %s for: %s",
|
||||||
self.import_handler.actioning,
|
self.import_handler,
|
||||||
self.import_handler.get_title(),
|
self.import_handler.orientation.value,
|
||||||
", ".join(models),
|
", ".join(models),
|
||||||
)
|
)
|
||||||
log.debug("params are: %s", kw)
|
log.debug("params are: %s", kw)
|
||||||
|
|
@ -146,16 +127,15 @@ class ImportCommandHandler(GenericHandler):
|
||||||
|
|
||||||
This is what happens when command line has ``--list-models``.
|
This is what happens when command line has ``--list-models``.
|
||||||
"""
|
"""
|
||||||
sys.stdout.write("\nALL MODELS:\n")
|
sys.stdout.write("ALL MODELS:\n")
|
||||||
sys.stdout.write("==============================\n")
|
sys.stdout.write("==============================\n")
|
||||||
for key in self.import_handler.importers:
|
for key in self.import_handler.importers:
|
||||||
sys.stdout.write(key)
|
sys.stdout.write(key)
|
||||||
sys.stdout.write("\n")
|
sys.stdout.write("\n")
|
||||||
sys.stdout.write("==============================\n")
|
sys.stdout.write("==============================\n")
|
||||||
sys.stdout.write(f"for {self.import_handler.get_title()}\n\n")
|
|
||||||
|
|
||||||
|
|
||||||
def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments,too-many-locals
|
def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments
|
||||||
models: Annotated[
|
models: Annotated[
|
||||||
Optional[List[str]],
|
Optional[List[str]],
|
||||||
typer.Argument(
|
typer.Argument(
|
||||||
|
|
@ -237,27 +217,6 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
|
||||||
help="Max number of *any* target record changes which may occur (per model)."
|
help="Max number of *any* target record changes which may occur (per model)."
|
||||||
),
|
),
|
||||||
] = None,
|
] = None,
|
||||||
warnings: Annotated[
|
|
||||||
bool,
|
|
||||||
typer.Option(
|
|
||||||
"--warn",
|
|
||||||
"-W",
|
|
||||||
help="Expect no changes; warn (email the diff) if any occur.",
|
|
||||||
),
|
|
||||||
] = False,
|
|
||||||
warnings_recipients: Annotated[
|
|
||||||
str,
|
|
||||||
typer.Option(
|
|
||||||
"--recip", help="Override the recipient(s) for diff warning email."
|
|
||||||
),
|
|
||||||
] = None,
|
|
||||||
warnings_max_diffs: Annotated[
|
|
||||||
int,
|
|
||||||
typer.Option(
|
|
||||||
"--max-diffs",
|
|
||||||
help="Max number of record diffs to show (per model) in warning email.",
|
|
||||||
),
|
|
||||||
] = 15,
|
|
||||||
dry_run: Annotated[
|
dry_run: Annotated[
|
||||||
bool,
|
bool,
|
||||||
typer.Option(
|
typer.Option(
|
||||||
|
|
|
||||||
|
|
@ -38,5 +38,7 @@ def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
|
||||||
Import data from CSV file(s) to Wutta DB
|
Import data from CSV file(s) to Wutta DB
|
||||||
"""
|
"""
|
||||||
config = ctx.parent.wutta_config
|
config = ctx.parent.wutta_config
|
||||||
handler = ImportCommandHandler(config, key="import.to_wutta.from_csv")
|
handler = ImportCommandHandler(
|
||||||
|
config, import_handler="wuttasync.importing.csv:FromCsvToWutta"
|
||||||
|
)
|
||||||
handler.run(ctx.params)
|
handler.run(ctx.params)
|
||||||
|
|
|
||||||
|
|
@ -1,73 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
################################################################################
|
|
||||||
#
|
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
|
||||||
#
|
|
||||||
# This file is part of Wutta Framework.
|
|
||||||
#
|
|
||||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
|
||||||
# under the terms of the GNU General Public License as published by the Free
|
|
||||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
|
||||||
# later version.
|
|
||||||
#
|
|
||||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
|
||||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
# more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License along with
|
|
||||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
################################################################################
|
|
||||||
"""
|
|
||||||
See also: :ref:`wutta-import-versions`
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
|
|
||||||
import rich
|
|
||||||
import typer
|
|
||||||
from typing_extensions import Annotated
|
|
||||||
|
|
||||||
from wuttjamaican.cli import wutta_typer
|
|
||||||
|
|
||||||
from .base import import_command, ImportCommandHandler
|
|
||||||
|
|
||||||
|
|
||||||
@wutta_typer.command()
|
|
||||||
@import_command
|
|
||||||
def import_versions( # pylint: disable=unused-argument
|
|
||||||
ctx: typer.Context,
|
|
||||||
comment: Annotated[
|
|
||||||
str,
|
|
||||||
typer.Option("--comment", "-m", help="Comment to set on the transaction."),
|
|
||||||
] = "import catch-up versions",
|
|
||||||
**kwargs,
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Import latest data to version tables, for Wutta DB
|
|
||||||
"""
|
|
||||||
config = ctx.parent.wutta_config
|
|
||||||
app = config.get_app()
|
|
||||||
|
|
||||||
# warn/exit if libs are not installed
|
|
||||||
try:
|
|
||||||
import wutta_continuum # pylint: disable=import-outside-toplevel,unused-import
|
|
||||||
except ImportError: # pragma: no cover
|
|
||||||
rich.print(
|
|
||||||
"\n\t[bold yellow]Wutta-Continum is not installed![/bold yellow]\n"
|
|
||||||
"\n\tIf you want it, run: pip install Wutta-Continuum\n"
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
# warn/exit if feature disabled
|
|
||||||
if not app.continuum_is_enabled(): # pragma: no cover
|
|
||||||
rich.print(
|
|
||||||
"\n\t[bold yellow]Wutta-Continum is not enabled![/bold yellow]\n"
|
|
||||||
"\n\tIf you want it, see: https://docs.wuttaproject.org/wutta-continuum/\n"
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
handler = ImportCommandHandler(config, key="import.to_versions.from_wutta")
|
|
||||||
handler.run(ctx.params)
|
|
||||||
|
|
@ -1,88 +0,0 @@
|
||||||
## -*- coding: utf-8; -*-
|
|
||||||
<html>
|
|
||||||
<body>
|
|
||||||
<h3>Diff warning for ${title} (${handler.actioning})</h3>
|
|
||||||
|
|
||||||
<p style="font-style: italic;">
|
|
||||||
% if dry_run:
|
|
||||||
<span style="font-weight: bold;">DRY RUN</span>
|
|
||||||
- these changes have not yet happened
|
|
||||||
% else:
|
|
||||||
<span style="font-weight: bold;">LIVE RUN</span>
|
|
||||||
- these changes already happened
|
|
||||||
% endif
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<ul>
|
|
||||||
% for model, (created, updated, deleted) in changes.items():
|
|
||||||
<li>
|
|
||||||
<a href="#${model}">${model}</a> -
|
|
||||||
${app.render_quantity(len(created))} created;
|
|
||||||
${app.render_quantity(len(updated))} updated;
|
|
||||||
${app.render_quantity(len(deleted))} deleted
|
|
||||||
</li>
|
|
||||||
% endfor
|
|
||||||
</ul>
|
|
||||||
|
|
||||||
<p>
|
|
||||||
<span style="font-weight: bold;">COMMAND:</span>
|
|
||||||
|
|
||||||
<code>${argv}</code>
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<p>
|
|
||||||
<span style="font-weight: bold;">RUNTIME:</span>
|
|
||||||
|
|
||||||
${runtime} (${runtime_display})
|
|
||||||
</p>
|
|
||||||
|
|
||||||
% for model, (created, updated, deleted) in changes.items():
|
|
||||||
|
|
||||||
<br />
|
|
||||||
<h4>
|
|
||||||
<a name="${model}">${model}</a> -
|
|
||||||
${app.render_quantity(len(created))} created;
|
|
||||||
${app.render_quantity(len(updated))} updated;
|
|
||||||
${app.render_quantity(len(deleted))} deleted
|
|
||||||
</h4>
|
|
||||||
|
|
||||||
<div style="padding-left: 2rem;">
|
|
||||||
|
|
||||||
% for obj, source_data in created[:max_diffs]:
|
|
||||||
<h5>${model} <em>created</em> in ${target_title}: ${obj}</h5>
|
|
||||||
<% diff = make_diff({}, source_data, nature="create") %>
|
|
||||||
<div style="padding-left: 2rem;">
|
|
||||||
${diff.render_html()}
|
|
||||||
</div>
|
|
||||||
% endfor
|
|
||||||
% if len(created) > max_diffs:
|
|
||||||
<h5>${model} - ${app.render_quantity(len(created) - max_diffs)} more records <em>created</em> in ${target_title} - not shown here</h5>
|
|
||||||
% endif
|
|
||||||
|
|
||||||
% for obj, source_data, target_data in updated[:max_diffs]:
|
|
||||||
<h5>${model} <em>updated</em> in ${target_title}: ${obj}</h5>
|
|
||||||
<% diff = make_diff(target_data, source_data, nature="update") %>
|
|
||||||
<div style="padding-left: 2rem;">
|
|
||||||
${diff.render_html()}
|
|
||||||
</div>
|
|
||||||
% endfor
|
|
||||||
% if len(updated) > max_diffs:
|
|
||||||
<h5>${model} - ${app.render_quantity(len(updated) - max_diffs)} more records <em>updated</em> in ${target_title} - not shown here</h5>
|
|
||||||
% endif
|
|
||||||
|
|
||||||
% for obj, target_data in deleted[:max_diffs]:
|
|
||||||
<h5>${model} <em>deleted</em> in ${target_title}: ${obj}</h5>
|
|
||||||
<% diff = make_diff(target_data, {}, nature="delete") %>
|
|
||||||
<div style="padding-left: 2rem;">
|
|
||||||
${diff.render_html()}
|
|
||||||
</div>
|
|
||||||
% endfor
|
|
||||||
% if len(deleted) > max_diffs:
|
|
||||||
<h5>${model} - ${app.render_quantity(len(deleted) - max_diffs)} more records <em>deleted</em> in ${target_title} - not shown here</h5>
|
|
||||||
% endif
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
% endfor
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
@ -1,166 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
################################################################################
|
|
||||||
#
|
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
|
||||||
#
|
|
||||||
# This file is part of Wutta Framework.
|
|
||||||
#
|
|
||||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
|
||||||
# under the terms of the GNU General Public License as published by the Free
|
|
||||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
|
||||||
# later version.
|
|
||||||
#
|
|
||||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
|
||||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
# more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License along with
|
|
||||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
################################################################################
|
|
||||||
"""
|
|
||||||
:term:`Email Settings <email setting>` for WuttaSync
|
|
||||||
"""
|
|
||||||
|
|
||||||
import datetime
|
|
||||||
import re
|
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from wuttjamaican.email import EmailSetting
|
|
||||||
from wuttjamaican.diffs import Diff
|
|
||||||
|
|
||||||
|
|
||||||
class ImportExportWarning(EmailSetting):
|
|
||||||
"""
|
|
||||||
Base class for import/export diff warnings; sent when unexpected
|
|
||||||
changes occur.
|
|
||||||
|
|
||||||
This inherits from :class:`~wuttjamaican.email.EmailSetting`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
fallback_key = "import_export_warning"
|
|
||||||
"" # suppress docs
|
|
||||||
|
|
||||||
import_handler_spec = None
|
|
||||||
import_handler_key = None
|
|
||||||
|
|
||||||
def get_description(self): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
handler = self.get_import_handler()
|
|
||||||
return f"Diff warning email for {handler.actioning} {handler.get_title()}"
|
|
||||||
|
|
||||||
def get_default_subject(self): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
handler = self.get_import_handler()
|
|
||||||
return f"Changes for {handler.get_title()}"
|
|
||||||
|
|
||||||
def get_import_handler(self): # pylint: disable=missing-function-docstring
|
|
||||||
|
|
||||||
# prefer explicit spec, if set
|
|
||||||
if self.import_handler_spec:
|
|
||||||
return self.app.load_object(self.import_handler_spec)(self.config)
|
|
||||||
|
|
||||||
# next try spec lookup, if key set
|
|
||||||
if self.import_handler_key:
|
|
||||||
return self.app.get_import_handler(self.import_handler_key, require=True)
|
|
||||||
|
|
||||||
# or maybe try spec lookup basd on setting class name
|
|
||||||
class_name = self.__class__.__name__
|
|
||||||
if match := re.match(
|
|
||||||
r"^(?P<action>import|export)_to_(?P<target>\S+)_from_(?P<source>\S+)_warning$",
|
|
||||||
class_name,
|
|
||||||
):
|
|
||||||
key = f"{match['action']}.to_{match['target']}.from_{match['source']}"
|
|
||||||
return self.app.get_import_handler(key, require=True)
|
|
||||||
|
|
||||||
raise ValueError(
|
|
||||||
"must set import_handler_spec (or import_handler_key) "
|
|
||||||
f"for email setting: {class_name}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# nb. this is just used for sample data
|
|
||||||
def make_diff(self, *args, **kwargs): # pylint: disable=missing-function-docstring
|
|
||||||
return Diff(self.config, *args, **kwargs)
|
|
||||||
|
|
||||||
def sample_data(self): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
model = self.app.model
|
|
||||||
handler = self.get_import_handler()
|
|
||||||
|
|
||||||
alice = model.User(username="alice")
|
|
||||||
bob = model.User(username="bob")
|
|
||||||
charlie = model.User(username="charlie")
|
|
||||||
|
|
||||||
runtime = datetime.timedelta(seconds=30)
|
|
||||||
return {
|
|
||||||
"handler": handler,
|
|
||||||
"title": handler.get_title(),
|
|
||||||
"source_title": handler.get_source_title(),
|
|
||||||
"target_title": handler.get_target_title(),
|
|
||||||
"runtime": runtime,
|
|
||||||
"runtime_display": "30 seconds",
|
|
||||||
"dry_run": True,
|
|
||||||
"argv": [
|
|
||||||
"bin/wutta",
|
|
||||||
"import-foo",
|
|
||||||
"User",
|
|
||||||
"--delete",
|
|
||||||
"--dry-run",
|
|
||||||
"-W",
|
|
||||||
],
|
|
||||||
"changes": {
|
|
||||||
"User": (
|
|
||||||
[
|
|
||||||
(
|
|
||||||
alice,
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"),
|
|
||||||
"username": "alice",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
(
|
|
||||||
bob,
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
|
||||||
"username": "bob",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
|
||||||
"username": "bobbie",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
(
|
|
||||||
charlie,
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"),
|
|
||||||
"username": "charlie",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
},
|
|
||||||
"make_diff": self.make_diff,
|
|
||||||
"max_diffs": 15,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class import_to_versions_from_wutta_warning( # pylint: disable=invalid-name
|
|
||||||
ImportExportWarning
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Diff warning for Wutta → Versions import.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
class import_to_wutta_from_csv_warning( # pylint: disable=invalid-name
|
|
||||||
ImportExportWarning
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Diff warning for CSV → Wutta import.
|
|
||||||
"""
|
|
||||||
|
|
@ -2,7 +2,7 @@
|
||||||
################################################################################
|
################################################################################
|
||||||
#
|
#
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
# Copyright © 2024 Lance Edgar
|
||||||
#
|
#
|
||||||
# This file is part of Wutta Framework.
|
# This file is part of Wutta Framework.
|
||||||
#
|
#
|
||||||
|
|
@ -32,7 +32,7 @@ And some :term:`import handler` base classes:
|
||||||
* :class:`~wuttasync.importing.handlers.ImportHandler`
|
* :class:`~wuttasync.importing.handlers.ImportHandler`
|
||||||
* :class:`~wuttasync.importing.handlers.FromFileHandler`
|
* :class:`~wuttasync.importing.handlers.FromFileHandler`
|
||||||
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
|
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
|
||||||
* :class:`~wuttasync.importing.handlers.ToWuttaHandler`
|
* :class:`~wuttasync.importing.wutta.ToWuttaHandler`
|
||||||
|
|
||||||
And some :term:`importer` base classes:
|
And some :term:`importer` base classes:
|
||||||
|
|
||||||
|
|
@ -42,12 +42,7 @@ And some :term:`importer` base classes:
|
||||||
* :class:`~wuttasync.importing.model.ToWutta`
|
* :class:`~wuttasync.importing.model.ToWutta`
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .handlers import (
|
from .handlers import Orientation, ImportHandler, FromFileHandler, ToSqlalchemyHandler
|
||||||
Orientation,
|
|
||||||
ImportHandler,
|
|
||||||
FromFileHandler,
|
|
||||||
ToSqlalchemyHandler,
|
|
||||||
ToWuttaHandler,
|
|
||||||
)
|
|
||||||
from .base import Importer, FromFile, ToSqlalchemy
|
from .base import Importer, FromFile, ToSqlalchemy
|
||||||
from .model import ToWutta
|
from .model import ToWutta
|
||||||
|
from .wutta import ToWuttaHandler
|
||||||
|
|
|
||||||
|
|
@ -184,19 +184,6 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
||||||
:meth:`get_target_cache()`.
|
:meth:`get_target_cache()`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
default_keys = None
|
|
||||||
"""
|
|
||||||
In certain edge cases, the importer class must declare its key
|
|
||||||
list without using :attr:`keys`.
|
|
||||||
|
|
||||||
(As of now this only happens with
|
|
||||||
:class:`~wuttasync.importing.versions.FromWuttaToVersions` which
|
|
||||||
must dynamically create importer classes.)
|
|
||||||
|
|
||||||
If applicable, this value is used as fallback for
|
|
||||||
:meth:`get_keys()`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
max_create = None
|
max_create = None
|
||||||
max_update = None
|
max_update = None
|
||||||
max_delete = None
|
max_delete = None
|
||||||
|
|
@ -336,54 +323,19 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
||||||
|
|
||||||
def get_keys(self):
|
def get_keys(self):
|
||||||
"""
|
"""
|
||||||
Retrieve the list of key field(s) for use with import/export.
|
Must return the key field(s) for use with import/export.
|
||||||
The result is cached, so the key list is only calculated once.
|
|
||||||
|
|
||||||
Many importers have just one key field, but we always assume a
|
|
||||||
key *list* - so this often is a list with just one field.
|
|
||||||
|
|
||||||
All fields in this list should also be found in the output for
|
All fields in this list should also be found in the output for
|
||||||
:meth:`get_fields()`.
|
:meth:`get_fields()`.
|
||||||
|
|
||||||
Many importers will declare this via :attr:`keys` (or
|
|
||||||
:attr:`key`) static attribute::
|
|
||||||
|
|
||||||
class SprocketImporter(Importer):
|
|
||||||
|
|
||||||
# nb. all these examples work the same
|
|
||||||
|
|
||||||
# 'keys' is the preferred attribute
|
|
||||||
keys = ("sprocket_id",) # <-- the "canonical" way
|
|
||||||
keys = ["sprocket_id"]
|
|
||||||
keys = "sprocket_id"
|
|
||||||
|
|
||||||
# 'key' is not preferred, but works
|
|
||||||
key = ("sprocket_id",)
|
|
||||||
key = "sprocket_id"
|
|
||||||
|
|
||||||
If neither ``keys`` nor ``key`` is set, as a special case
|
|
||||||
:attr:`default_keys` is used if set.
|
|
||||||
|
|
||||||
If no keys were declared, the list is inspected from the model
|
|
||||||
class via
|
|
||||||
:func:`sqlalchemy-utils:sqlalchemy_utils.functions.get_primary_keys()`.
|
|
||||||
|
|
||||||
In any case, the determination is made only once. This method
|
|
||||||
also *sets* :attr:`keys` on the instance, so it will return
|
|
||||||
that as-is for subsequent calls.
|
|
||||||
|
|
||||||
:returns: List of "key" field names.
|
:returns: List of "key" field names.
|
||||||
"""
|
"""
|
||||||
keys = None
|
keys = None
|
||||||
|
|
||||||
# nb. prefer 'keys' but use 'key' as fallback
|
# nb. prefer 'keys' but use 'key' as fallback
|
||||||
if "keys" in self.__dict__:
|
if "keys" in self.__dict__:
|
||||||
keys = self.__dict__["keys"]
|
keys = self.__dict__["keys"]
|
||||||
elif "key" in self.__dict__:
|
elif "key" in self.__dict__:
|
||||||
keys = self.__dict__["key"]
|
keys = self.__dict__["key"]
|
||||||
else:
|
|
||||||
keys = self.default_keys
|
|
||||||
|
|
||||||
if keys:
|
if keys:
|
||||||
if isinstance(keys, str):
|
if isinstance(keys, str):
|
||||||
keys = self.config.parse_list(keys)
|
keys = self.config.parse_list(keys)
|
||||||
|
|
@ -459,10 +411,7 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
||||||
updated = []
|
updated = []
|
||||||
deleted = []
|
deleted = []
|
||||||
|
|
||||||
model_title = self.get_model_title()
|
log.debug("using key fields: %s", ", ".join(self.get_keys()))
|
||||||
log.debug(
|
|
||||||
"using key fields for %s: %s", model_title, ", ".join(self.get_keys())
|
|
||||||
)
|
|
||||||
|
|
||||||
# get complete set of normalized source data
|
# get complete set of normalized source data
|
||||||
if source_data is None:
|
if source_data is None:
|
||||||
|
|
@ -471,7 +420,8 @@ class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-
|
||||||
# nb. prune duplicate records from source data
|
# nb. prune duplicate records from source data
|
||||||
source_data, source_keys = self.get_unique_data(source_data)
|
source_data, source_keys = self.get_unique_data(source_data)
|
||||||
|
|
||||||
log.debug("got %s %s records from source", len(source_data), model_title)
|
model_title = self.get_model_title()
|
||||||
|
log.debug(f"got %s {model_title} records from source", len(source_data))
|
||||||
|
|
||||||
# maybe cache existing target data
|
# maybe cache existing target data
|
||||||
if self.caches_target:
|
if self.caches_target:
|
||||||
|
|
@ -1321,139 +1271,10 @@ class FromFile(Importer):
|
||||||
self.input_file.close()
|
self.input_file.close()
|
||||||
|
|
||||||
|
|
||||||
class QueryWrapper:
|
|
||||||
"""
|
|
||||||
Simple wrapper for a SQLAlchemy query, to make it sort of behave
|
|
||||||
so that an importer can treat it as a data record list.
|
|
||||||
|
|
||||||
:param query: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, query):
|
|
||||||
self.query = query
|
|
||||||
|
|
||||||
def __len__(self):
|
|
||||||
try:
|
|
||||||
return len(self.query)
|
|
||||||
except TypeError:
|
|
||||||
return self.query.count()
|
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
return iter(self.query)
|
|
||||||
|
|
||||||
|
|
||||||
class FromSqlalchemy(Importer): # pylint: disable=abstract-method
|
|
||||||
"""
|
|
||||||
Base class for importer/exporter using SQL/ORM query as data
|
|
||||||
source.
|
|
||||||
|
|
||||||
Subclass should define :attr:`source_model_class` in which case
|
|
||||||
the source query is automatic. And/or override
|
|
||||||
:meth:`get_source_query()` to customize.
|
|
||||||
|
|
||||||
See also :class:`FromSqlalchemyMirror` and :class:`ToSqlalchemy`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
source_model_class = None
|
|
||||||
"""
|
|
||||||
Reference to the :term:`data model` class representing the source.
|
|
||||||
|
|
||||||
This normally is a SQLAlchemy mapped class, e.g.
|
|
||||||
:class:`~wuttjamaican:wuttjamaican.db.model.base.Person` for
|
|
||||||
exporting from the Wutta People table.
|
|
||||||
"""
|
|
||||||
|
|
||||||
source_session = None
|
|
||||||
"""
|
|
||||||
Reference to the open :term:`db session` for the data source.
|
|
||||||
|
|
||||||
The importer must be given this reference when instantiated by the
|
|
||||||
:term:`import handler`. This is handled automatically if using
|
|
||||||
:class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def get_source_objects(self):
|
|
||||||
"""
|
|
||||||
This method is responsible for fetching "raw" (non-normalized)
|
|
||||||
records from data source.
|
|
||||||
|
|
||||||
(See also the parent method docs for
|
|
||||||
:meth:`~wuttasync.importing.base.Importer.get_source_objects()`.)
|
|
||||||
|
|
||||||
It calls :meth:`get_source_query()` and then wraps that in a
|
|
||||||
:class:`QueryWrapper`, which is then returned.
|
|
||||||
|
|
||||||
Note that this method does not technically "retrieve" records
|
|
||||||
from the query; that happens automatically later.
|
|
||||||
|
|
||||||
:returns: :class:`QueryWrapper` for the source query
|
|
||||||
"""
|
|
||||||
query = self.get_source_query()
|
|
||||||
return QueryWrapper(query)
|
|
||||||
|
|
||||||
def get_source_query(self):
|
|
||||||
"""
|
|
||||||
This returns the SQL/ORM query used to fetch source
|
|
||||||
data. It is called from :meth:`get_source_objects()`.
|
|
||||||
|
|
||||||
Default logic just makes a simple ``SELECT * FROM TABLE`` kind
|
|
||||||
of query. Subclass can override as needed.
|
|
||||||
|
|
||||||
:returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
|
|
||||||
"""
|
|
||||||
return self.source_session.query(self.source_model_class)
|
|
||||||
|
|
||||||
|
|
||||||
class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method
|
|
||||||
"""
|
|
||||||
Special base class for when the source and target are effectively
|
|
||||||
mirrored, and can each be represented by the same :term:`data
|
|
||||||
model`.
|
|
||||||
|
|
||||||
The assumption is that SQLAlchemy ORM is used on both sides, even
|
|
||||||
though this base class only defines the source side (it inherits
|
|
||||||
from :class:`FromSqlalchemy`).
|
|
||||||
|
|
||||||
There are two main use cases for this:
|
|
||||||
|
|
||||||
* sync between app nodes
|
|
||||||
* sync version tables
|
|
||||||
|
|
||||||
When 2 app nodes are synced, the source and target are "the same"
|
|
||||||
in a schema sense, e.g. ``sprockets on node 01 => sprockets on
|
|
||||||
node 02``.
|
|
||||||
|
|
||||||
When version tables are synced, the same schema can be used for
|
|
||||||
the "live" table and the "version" table, e.g. ``sprockets =>
|
|
||||||
sprocket versions``.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@property
|
|
||||||
def source_model_class(self):
|
|
||||||
"""
|
|
||||||
This returns the :attr:`~Importer.model_class` since source
|
|
||||||
and target must share common schema.
|
|
||||||
"""
|
|
||||||
return self.model_class
|
|
||||||
|
|
||||||
def normalize_source_object(self, obj):
|
|
||||||
"""
|
|
||||||
Since source/target share schema, there should be no tricky
|
|
||||||
normalization involved.
|
|
||||||
|
|
||||||
This calls :meth:`~Importer.normalize_target_object()` since
|
|
||||||
that logic should already be defined. This ensures the same
|
|
||||||
normalization is used on both sides.
|
|
||||||
"""
|
|
||||||
return self.normalize_target_object(obj)
|
|
||||||
|
|
||||||
|
|
||||||
class ToSqlalchemy(Importer):
|
class ToSqlalchemy(Importer):
|
||||||
"""
|
"""
|
||||||
Base class for importer/exporter which uses SQLAlchemy ORM on the
|
Base class for importer/exporter which uses SQLAlchemy ORM on the
|
||||||
target side.
|
target side.
|
||||||
|
|
||||||
See also :class:`FromSqlalchemy`.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
caches_target = True
|
caches_target = True
|
||||||
|
|
@ -1491,8 +1312,6 @@ class ToSqlalchemy(Importer):
|
||||||
Returns an ORM query suitable to fetch existing objects from
|
Returns an ORM query suitable to fetch existing objects from
|
||||||
the target side. This is called from
|
the target side. This is called from
|
||||||
:meth:`get_target_objects()`.
|
:meth:`get_target_objects()`.
|
||||||
|
|
||||||
:returns: :class:`~sqlalchemy:sqlalchemy.orm.Query` instance
|
|
||||||
"""
|
"""
|
||||||
return self.target_session.query(self.model_class)
|
return self.target_session.query(self.model_class)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,8 @@ from wuttjamaican.db.util import make_topo_sortkey, UUID
|
||||||
from wuttjamaican.util import parse_bool
|
from wuttjamaican.util import parse_bool
|
||||||
|
|
||||||
from .base import FromFile
|
from .base import FromFile
|
||||||
from .handlers import FromFileHandler, ToWuttaHandler
|
from .handlers import FromFileHandler
|
||||||
|
from .wutta import ToWuttaHandler
|
||||||
from .model import ToWutta
|
from .model import ToWutta
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -238,8 +239,6 @@ class FromCsvToSqlalchemyHandlerMixin:
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
# TODO: pylint (correctly) flags this as duplicate code, matching
|
|
||||||
# on the wuttasync.importing.versions module - should fix?
|
|
||||||
def define_importers(self):
|
def define_importers(self):
|
||||||
"""
|
"""
|
||||||
This mixin overrides typical (manual) importer definition, and
|
This mixin overrides typical (manual) importer definition, and
|
||||||
|
|
@ -253,7 +252,6 @@ class FromCsvToSqlalchemyHandlerMixin:
|
||||||
importers = {}
|
importers = {}
|
||||||
model = self.get_target_model()
|
model = self.get_target_model()
|
||||||
|
|
||||||
# pylint: disable=duplicate-code
|
|
||||||
# mostly try to make an importer for every data model
|
# mostly try to make an importer for every data model
|
||||||
for name in dir(model):
|
for name in dir(model):
|
||||||
cls = getattr(model, name)
|
cls = getattr(model, name)
|
||||||
|
|
|
||||||
|
|
@ -26,14 +26,10 @@ Data Import / Export Handlers
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
import humanize
|
|
||||||
|
|
||||||
from wuttjamaican.app import GenericHandler
|
from wuttjamaican.app import GenericHandler
|
||||||
from wuttjamaican.diffs import Diff
|
|
||||||
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
@ -48,7 +44,7 @@ class Orientation(Enum):
|
||||||
EXPORT = "export"
|
EXPORT = "export"
|
||||||
|
|
||||||
|
|
||||||
class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
class ImportHandler(GenericHandler):
|
||||||
"""
|
"""
|
||||||
Base class for all import/export handlers.
|
Base class for all import/export handlers.
|
||||||
|
|
||||||
|
|
@ -125,47 +121,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
:meth:`commit_transaction()`.
|
:meth:`commit_transaction()`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
process_started = None
|
|
||||||
|
|
||||||
warnings = False
|
|
||||||
"""
|
|
||||||
Boolean indicating the import/export should run in "warnings"
|
|
||||||
mode.
|
|
||||||
|
|
||||||
If set, this declares that no changes are expected for the
|
|
||||||
import/export job. If any changes do occur with this flag set, a
|
|
||||||
diff warning email is sent within :meth:`process_changes()`.
|
|
||||||
|
|
||||||
See also :attr:`warnings_recipients`,
|
|
||||||
:attr:`warnings_max_diffs` and :attr:`warnings_email_key`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
warnings_email_key = None
|
|
||||||
"""
|
|
||||||
Explicit :term:`email key` for sending the diff warning email,
|
|
||||||
*unique to this import/export type*.
|
|
||||||
|
|
||||||
Handlers do not normally set this, so the email key is determined
|
|
||||||
automatically within :meth:`get_warnings_email_key()`.
|
|
||||||
|
|
||||||
See also :attr:`warnings`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
warnings_recipients = None
|
|
||||||
"""
|
|
||||||
Explicit recipient list for the warning email. If not set, the
|
|
||||||
recipients are determined automatically via config.
|
|
||||||
|
|
||||||
See also :attr:`warnings`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
warnings_max_diffs = 15
|
|
||||||
"""
|
|
||||||
Max number of record diffs (per model) to show in the warning email.
|
|
||||||
|
|
||||||
See also :attr:`warnings`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
importers = None
|
importers = None
|
||||||
"""
|
"""
|
||||||
This should be a dict of all importer/exporter classes available
|
This should be a dict of all importer/exporter classes available
|
||||||
|
|
@ -209,21 +164,18 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_key(cls):
|
def get_key(cls):
|
||||||
"""
|
"""
|
||||||
Returns the :term:`import/export key` for the handler. This
|
Returns the "full key" for the handler. This is a combination
|
||||||
is a combination of :attr:`source_key` and :attr:`target_key`
|
of :attr:`source_key` and :attr:`target_key` and
|
||||||
and :attr:`orientation`.
|
:attr:`orientation`.
|
||||||
|
|
||||||
For instance in the case of Wutta → CSV export, the key is:
|
For instance in the case of CSV → Wutta, the full handler key
|
||||||
``export.to_csv.from_wutta``
|
is ``to_wutta.from_csv.import``.
|
||||||
|
|
||||||
Note that more than one handler may use the same key; but only
|
Note that more than one handler may return the same full key
|
||||||
one will be configured as the "designated" handler for that
|
here; but only one will be configured as the "default" handler
|
||||||
key, a la
|
for that key. See also :meth:`get_spec()`.
|
||||||
:meth:`~wuttasync.app.WuttaSyncAppProvider.get_import_handler()`.
|
|
||||||
|
|
||||||
See also :meth:`get_spec()`.
|
|
||||||
"""
|
"""
|
||||||
return f"{cls.orientation.value}.to_{cls.target_key}.from_{cls.source_key}"
|
return f"to_{cls.target_key}.from_{cls.source_key}.{cls.orientation.value}"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_spec(cls):
|
def get_spec(cls):
|
||||||
|
|
@ -257,20 +209,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
"""
|
"""
|
||||||
Returns the display title for the data source.
|
Returns the display title for the data source.
|
||||||
|
|
||||||
By default this returns :attr:`source_key`, but this can be
|
|
||||||
overriden by class attribute.
|
|
||||||
|
|
||||||
Base class can define ``generic_source_title`` to provide a
|
|
||||||
new default::
|
|
||||||
|
|
||||||
class FromExcelHandler(ImportHandler):
|
|
||||||
generic_source_title = "Excel File"
|
|
||||||
|
|
||||||
Subclass can define ``source_title`` to be explicit::
|
|
||||||
|
|
||||||
class FromExcelToWutta(FromExcelHandler, ToWuttaHandler):
|
|
||||||
source_title = "My Spreadsheet"
|
|
||||||
|
|
||||||
See also :meth:`get_title()` and :meth:`get_target_title()`.
|
See also :meth:`get_title()` and :meth:`get_target_title()`.
|
||||||
"""
|
"""
|
||||||
if hasattr(self, "source_title"):
|
if hasattr(self, "source_title"):
|
||||||
|
|
@ -283,20 +221,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
"""
|
"""
|
||||||
Returns the display title for the data target.
|
Returns the display title for the data target.
|
||||||
|
|
||||||
By default this returns :attr:`target_key`, but this can be
|
|
||||||
overriden by class attribute.
|
|
||||||
|
|
||||||
Base class can define ``generic_target_title`` to provide a
|
|
||||||
new default::
|
|
||||||
|
|
||||||
class ToExcelHandler(ImportHandler):
|
|
||||||
generic_target_title = "Excel File"
|
|
||||||
|
|
||||||
Subclass can define ``target_title`` to be explicit::
|
|
||||||
|
|
||||||
class FromWuttaToExcel(FromWuttaHandler, ToExcelHandler):
|
|
||||||
target_title = "My Spreadsheet"
|
|
||||||
|
|
||||||
See also :meth:`get_title()` and :meth:`get_source_title()`.
|
See also :meth:`get_title()` and :meth:`get_source_title()`.
|
||||||
"""
|
"""
|
||||||
if hasattr(self, "target_title"):
|
if hasattr(self, "target_title"):
|
||||||
|
|
@ -326,17 +250,11 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
* :meth:`begin_transaction()`
|
* :meth:`begin_transaction()`
|
||||||
* :meth:`get_importer()`
|
* :meth:`get_importer()`
|
||||||
* :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter)
|
* :meth:`~wuttasync.importing.base.Importer.process_data()` (on the importer/exporter)
|
||||||
* :meth:`process_changes()`
|
|
||||||
* :meth:`rollback_transaction()`
|
* :meth:`rollback_transaction()`
|
||||||
* :meth:`commit_transaction()`
|
* :meth:`commit_transaction()`
|
||||||
"""
|
"""
|
||||||
kwargs = self.consume_kwargs(kwargs)
|
kwargs = self.consume_kwargs(kwargs)
|
||||||
self.process_started = self.app.localtime()
|
|
||||||
self.begin_transaction()
|
self.begin_transaction()
|
||||||
changes = OrderedDict()
|
|
||||||
|
|
||||||
if not keys:
|
|
||||||
keys = list(self.importers)
|
|
||||||
|
|
||||||
success = False
|
success = False
|
||||||
try:
|
try:
|
||||||
|
|
@ -347,31 +265,22 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
# invoke importer
|
# invoke importer
|
||||||
importer = self.get_importer(key, **kwargs)
|
importer = self.get_importer(key, **kwargs)
|
||||||
created, updated, deleted = importer.process_data()
|
created, updated, deleted = importer.process_data()
|
||||||
changed = bool(created or updated or deleted)
|
|
||||||
|
|
||||||
# log what happened
|
# log what happened
|
||||||
msg = "%s: added %d; updated %d; deleted %d %s records"
|
msg = "%s: added %d; updated %d; deleted %d %s records"
|
||||||
if self.dry_run:
|
if self.dry_run:
|
||||||
msg += " (dry run)"
|
msg += " (dry run)"
|
||||||
logger = log.warning if changed and self.warnings else log.info
|
log.info(
|
||||||
logger(
|
|
||||||
msg, self.get_title(), len(created), len(updated), len(deleted), key
|
msg, self.get_title(), len(created), len(updated), len(deleted), key
|
||||||
)
|
)
|
||||||
|
|
||||||
# keep track of any changes
|
|
||||||
if changed:
|
|
||||||
changes[key] = created, updated, deleted
|
|
||||||
|
|
||||||
# post-processing for all changes
|
|
||||||
if changes:
|
|
||||||
self.process_changes(changes)
|
|
||||||
|
|
||||||
success = True
|
|
||||||
|
|
||||||
except:
|
except:
|
||||||
log.exception("what should happen here?") # TODO
|
log.exception("what should happen here?") # TODO
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
else:
|
||||||
|
success = True
|
||||||
|
|
||||||
finally:
|
finally:
|
||||||
if not success:
|
if not success:
|
||||||
log.warning("something failed, so transaction was rolled back")
|
log.warning("something failed, so transaction was rolled back")
|
||||||
|
|
@ -405,17 +314,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
if "dry_run" in kwargs:
|
if "dry_run" in kwargs:
|
||||||
self.dry_run = kwargs["dry_run"]
|
self.dry_run = kwargs["dry_run"]
|
||||||
|
|
||||||
if "warnings" in kwargs:
|
|
||||||
self.warnings = kwargs.pop("warnings")
|
|
||||||
|
|
||||||
if "warnings_recipients" in kwargs:
|
|
||||||
self.warnings_recipients = self.config.parse_list(
|
|
||||||
kwargs.pop("warnings_recipients")
|
|
||||||
)
|
|
||||||
|
|
||||||
if "warnings_max_diffs" in kwargs:
|
|
||||||
self.warnings_max_diffs = kwargs.pop("warnings_max_diffs")
|
|
||||||
|
|
||||||
return kwargs
|
return kwargs
|
||||||
|
|
||||||
def begin_transaction(self):
|
def begin_transaction(self):
|
||||||
|
|
@ -614,113 +512,6 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||||
"""
|
"""
|
||||||
return kwargs
|
return kwargs
|
||||||
|
|
||||||
def process_changes(self, changes):
|
|
||||||
"""
|
|
||||||
Run post-processing operations on the given changes, if
|
|
||||||
applicable.
|
|
||||||
|
|
||||||
This method is called by :meth:`process_data()`, if any
|
|
||||||
changes were made.
|
|
||||||
|
|
||||||
Default logic will send a "diff warning" email to the
|
|
||||||
configured recipient(s), if :attr:`warnings` mode is enabled.
|
|
||||||
If it is not enabled, nothing happens.
|
|
||||||
|
|
||||||
:param changes: :class:`~python:collections.OrderedDict` of
|
|
||||||
changes from the overall import/export job. The structure
|
|
||||||
is described below.
|
|
||||||
|
|
||||||
Keys for the ``changes`` dict will be model/importer names,
|
|
||||||
for instance::
|
|
||||||
|
|
||||||
{
|
|
||||||
"Sprocket": {...},
|
|
||||||
"User": {...},
|
|
||||||
}
|
|
||||||
|
|
||||||
Value for each model key is a 3-tuple of ``(created, updated,
|
|
||||||
deleted)``. Each of those elements is a list::
|
|
||||||
|
|
||||||
{
|
|
||||||
"Sprocket": (
|
|
||||||
[...], # created
|
|
||||||
[...], # updated
|
|
||||||
[...], # deleted
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
The list elements are always tuples, but the structure
|
|
||||||
varies::
|
|
||||||
|
|
||||||
{
|
|
||||||
"Sprocket": (
|
|
||||||
[ # created, 2-tuples
|
|
||||||
(obj, source_data),
|
|
||||||
],
|
|
||||||
[ # updated, 3-tuples
|
|
||||||
(obj, source_data, target_data),
|
|
||||||
],
|
|
||||||
[ # deleted, 2-tuples
|
|
||||||
(obj, target_data),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
if not self.warnings:
|
|
||||||
return
|
|
||||||
|
|
||||||
def make_diff(*args, **kwargs):
|
|
||||||
return Diff(self.config, *args, **kwargs)
|
|
||||||
|
|
||||||
runtime = self.app.localtime() - self.process_started
|
|
||||||
data = {
|
|
||||||
"handler": self,
|
|
||||||
"title": self.get_title(),
|
|
||||||
"source_title": self.get_source_title(),
|
|
||||||
"target_title": self.get_target_title(),
|
|
||||||
"dry_run": self.dry_run,
|
|
||||||
"argv": sys.argv,
|
|
||||||
"runtime": runtime,
|
|
||||||
"runtime_display": humanize.naturaldelta(runtime),
|
|
||||||
"changes": changes,
|
|
||||||
"make_diff": make_diff,
|
|
||||||
"max_diffs": self.warnings_max_diffs,
|
|
||||||
}
|
|
||||||
|
|
||||||
# maybe override recipients
|
|
||||||
kw = {}
|
|
||||||
if self.warnings_recipients:
|
|
||||||
kw["to"] = self.warnings_recipients
|
|
||||||
# TODO: should we in fact clear these..?
|
|
||||||
kw["cc"] = []
|
|
||||||
kw["bcc"] = []
|
|
||||||
|
|
||||||
# send the email
|
|
||||||
email_key = self.get_warnings_email_key()
|
|
||||||
self.app.send_email(email_key, data, fallback_key="import_export_warning", **kw)
|
|
||||||
|
|
||||||
log.info("%s: warning email was sent", self.get_title())
|
|
||||||
|
|
||||||
def get_warnings_email_key(self):
|
|
||||||
"""
|
|
||||||
Returns the :term:`email key` to be used for sending the diff
|
|
||||||
warning email.
|
|
||||||
|
|
||||||
The email key should be unique to this import/export type
|
|
||||||
(really, the :term:`import/export key`) but not necessarily
|
|
||||||
unique to one handler.
|
|
||||||
|
|
||||||
If :attr:`warnings_email_key` is set, it will be used as-is.
|
|
||||||
|
|
||||||
Otherwise one is generated from :meth:`get_key()`.
|
|
||||||
|
|
||||||
:returns: Email key for diff warnings
|
|
||||||
"""
|
|
||||||
if self.warnings_email_key:
|
|
||||||
return self.warnings_email_key
|
|
||||||
|
|
||||||
return self.get_key().replace(".", "_") + "_warning"
|
|
||||||
|
|
||||||
|
|
||||||
class FromFileHandler(ImportHandler):
|
class FromFileHandler(ImportHandler):
|
||||||
"""
|
"""
|
||||||
|
|
@ -747,129 +538,9 @@ class FromFileHandler(ImportHandler):
|
||||||
super().process_data(*keys, **kwargs)
|
super().process_data(*keys, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
class FromSqlalchemyHandler(ImportHandler):
|
|
||||||
"""
|
|
||||||
Base class for import/export handlers using SQLAlchemy ORM (DB) as
|
|
||||||
data source.
|
|
||||||
|
|
||||||
This is meant to be used with importers/exporters which inherit
|
|
||||||
from :class:`~wuttasync.importing.base.FromSqlalchemy`. It will
|
|
||||||
set the
|
|
||||||
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_session`
|
|
||||||
attribute when making them; cf. :meth:`get_importer_kwargs()`.
|
|
||||||
|
|
||||||
This is the base class for :class:`FromWuttaHandler`, but can be
|
|
||||||
used with any database.
|
|
||||||
|
|
||||||
See also :class:`ToSqlalchemyHandler`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
source_session = None
|
|
||||||
"""
|
|
||||||
Reference to the :term:`db session` for data source.
|
|
||||||
|
|
||||||
This will be ``None`` unless a transaction is running.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def begin_source_transaction(self):
|
|
||||||
"""
|
|
||||||
This calls :meth:`make_source_session()` and assigns the
|
|
||||||
result to :attr:`source_session`.
|
|
||||||
"""
|
|
||||||
self.source_session = self.make_source_session()
|
|
||||||
|
|
||||||
def commit_source_transaction(self):
|
|
||||||
"""
|
|
||||||
This commits and closes :attr:`source_session`.
|
|
||||||
"""
|
|
||||||
self.source_session.commit()
|
|
||||||
self.source_session.close()
|
|
||||||
self.source_session = None
|
|
||||||
|
|
||||||
def rollback_source_transaction(self):
|
|
||||||
"""
|
|
||||||
This rolls back, then closes :attr:`source_session`.
|
|
||||||
"""
|
|
||||||
self.source_session.rollback()
|
|
||||||
self.source_session.close()
|
|
||||||
self.source_session = None
|
|
||||||
|
|
||||||
def make_source_session(self):
|
|
||||||
"""
|
|
||||||
Make and return a new :term:`db session` for the data source.
|
|
||||||
|
|
||||||
Default logic is not implemented; subclass must override.
|
|
||||||
|
|
||||||
:returns: :class:`~sqlalchemy.orm.Session` instance
|
|
||||||
"""
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
def get_importer_kwargs(self, key, **kwargs):
|
|
||||||
"""
|
|
||||||
This modifies the new importer kwargs to add:
|
|
||||||
|
|
||||||
* ``source_session`` - reference to :attr:`source_session`
|
|
||||||
|
|
||||||
See also docs for parent method,
|
|
||||||
:meth:`~ImportHandler.get_importer_kwargs()`.
|
|
||||||
"""
|
|
||||||
kwargs = super().get_importer_kwargs(key, **kwargs)
|
|
||||||
kwargs["source_session"] = self.source_session
|
|
||||||
return kwargs
|
|
||||||
|
|
||||||
|
|
||||||
class FromWuttaHandler(FromSqlalchemyHandler):
|
|
||||||
"""
|
|
||||||
Handler for import/export which uses Wutta ORM (:term:`app
|
|
||||||
database`) as data source.
|
|
||||||
|
|
||||||
This inherits from :class:`FromSqlalchemyHandler`.
|
|
||||||
|
|
||||||
See also :class:`ToWuttaHandler`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
source_key = "wutta"
|
|
||||||
"" # nb. suppress docs
|
|
||||||
|
|
||||||
def get_source_title(self):
|
|
||||||
"""
|
|
||||||
This overrides default logic to use
|
|
||||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.get_title()`
|
|
||||||
as the default value.
|
|
||||||
|
|
||||||
Subclass can still define
|
|
||||||
:attr:`~wuttasync.importing.handlers.ImportHandler.source_title`
|
|
||||||
(or
|
|
||||||
:attr:`~wuttasync.importing.handlers.ImportHandler.generic_source_title`)
|
|
||||||
to customize.
|
|
||||||
|
|
||||||
See also docs for parent method:
|
|
||||||
:meth:`~wuttasync.importing.handlers.ImportHandler.get_source_title()`
|
|
||||||
"""
|
|
||||||
if hasattr(self, "source_title"):
|
|
||||||
return self.source_title
|
|
||||||
if hasattr(self, "generic_source_title"):
|
|
||||||
return self.generic_source_title
|
|
||||||
return self.app.get_title()
|
|
||||||
|
|
||||||
def make_source_session(self):
|
|
||||||
"""
|
|
||||||
This calls
|
|
||||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
|
||||||
and returns it.
|
|
||||||
"""
|
|
||||||
return self.app.make_session()
|
|
||||||
|
|
||||||
|
|
||||||
class ToSqlalchemyHandler(ImportHandler):
|
class ToSqlalchemyHandler(ImportHandler):
|
||||||
"""
|
"""
|
||||||
Base class for import/export handlers which target a SQLAlchemy
|
Handler for import/export which targets a SQLAlchemy ORM (DB).
|
||||||
ORM (DB).
|
|
||||||
|
|
||||||
This is the base class for :class:`ToWuttaHandler`, but can be
|
|
||||||
used with any database.
|
|
||||||
|
|
||||||
See also :class:`FromSqlalchemyHandler`.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
target_session = None
|
target_session = None
|
||||||
|
|
@ -920,37 +591,3 @@ class ToSqlalchemyHandler(ImportHandler):
|
||||||
kwargs = super().get_importer_kwargs(key, **kwargs)
|
kwargs = super().get_importer_kwargs(key, **kwargs)
|
||||||
kwargs.setdefault("target_session", self.target_session)
|
kwargs.setdefault("target_session", self.target_session)
|
||||||
return kwargs
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
class ToWuttaHandler(ToSqlalchemyHandler):
|
|
||||||
"""
|
|
||||||
Handler for import/export which targets Wutta ORM (:term:`app
|
|
||||||
database`).
|
|
||||||
|
|
||||||
This inherits from :class:`ToSqlalchemyHandler`.
|
|
||||||
|
|
||||||
See also :class:`FromWuttaHandler`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
target_key = "wutta"
|
|
||||||
"" # nb. suppress docs
|
|
||||||
|
|
||||||
def get_target_title(self): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
# nb. we override parent to use app title as default
|
|
||||||
if hasattr(self, "target_title"):
|
|
||||||
return self.target_title
|
|
||||||
if hasattr(self, "generic_target_title"):
|
|
||||||
return self.generic_target_title
|
|
||||||
return self.app.get_title()
|
|
||||||
|
|
||||||
def make_target_session(self):
|
|
||||||
"""
|
|
||||||
Call
|
|
||||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
|
||||||
and return the result.
|
|
||||||
|
|
||||||
:returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
|
|
||||||
instance.
|
|
||||||
"""
|
|
||||||
return self.app.make_session()
|
|
||||||
|
|
|
||||||
|
|
@ -1,353 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
################################################################################
|
|
||||||
#
|
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
|
||||||
#
|
|
||||||
# This file is part of Wutta Framework.
|
|
||||||
#
|
|
||||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
|
||||||
# under the terms of the GNU General Public License as published by the Free
|
|
||||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
|
||||||
# later version.
|
|
||||||
#
|
|
||||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
|
||||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
# more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License along with
|
|
||||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
################################################################################
|
|
||||||
"""
|
|
||||||
Importing Versions
|
|
||||||
|
|
||||||
This is a special type of import, only relevant when data versioning
|
|
||||||
is enabled.
|
|
||||||
|
|
||||||
See the handler class for more info: :class:`FromWuttaToVersions`
|
|
||||||
"""
|
|
||||||
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
from sqlalchemy_utils.functions import get_primary_keys
|
|
||||||
|
|
||||||
from wuttjamaican.db.util import make_topo_sortkey
|
|
||||||
|
|
||||||
from .handlers import FromWuttaHandler, ToWuttaHandler
|
|
||||||
from .wutta import FromWuttaMirror
|
|
||||||
from .model import ToWutta
|
|
||||||
|
|
||||||
|
|
||||||
class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
|
|
||||||
"""
|
|
||||||
Handler for Wutta -> Versions import.
|
|
||||||
|
|
||||||
The purpose of this is to ensure version tables accurately reflect
|
|
||||||
the current "live" data set, for given table(s). It is only
|
|
||||||
relevant/usable if versioning is configured and enabled. For more
|
|
||||||
on that see :doc:`wutta-continuum:index`.
|
|
||||||
|
|
||||||
For a given import model, the source is the "live" table, target
|
|
||||||
is the "version" table - both in the same :term:`app database`.
|
|
||||||
|
|
||||||
When reading data from the target side, it only grabs the "latest"
|
|
||||||
(valid) version record for each comparison to source.
|
|
||||||
|
|
||||||
When changes are needed, instead of updating the existing version
|
|
||||||
record, it always writes a new version record.
|
|
||||||
|
|
||||||
This handler will dynamically create importers for all versioned
|
|
||||||
models in the :term:`app model`; see
|
|
||||||
:meth:`make_importer_factory()`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
target_key = "versions"
|
|
||||||
target_title = "Versions"
|
|
||||||
|
|
||||||
continuum_uow = None
|
|
||||||
"""
|
|
||||||
Reference to the
|
|
||||||
:class:`sqlalchemy-continuum:`sqlalchemy_continuum.UnitOfWork`
|
|
||||||
created (by the SQLAlchemy-Continuum ``versioning_manager``) when
|
|
||||||
the transaction begins.
|
|
||||||
|
|
||||||
See also :attr:`continuum_txn` and
|
|
||||||
:meth:`begin_target_transaction()`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
continuum_txn = None
|
|
||||||
"""
|
|
||||||
Reference to the SQLAlchemy-Continuum ``transaction`` record, to
|
|
||||||
which any new version records will associate (if needed).
|
|
||||||
|
|
||||||
This transaction will track the effective user responsible for
|
|
||||||
the change(s), their client IP, and timestamp.
|
|
||||||
|
|
||||||
This reference is passed along to the importers as well (as
|
|
||||||
:attr:`~FromWuttaToVersionBase.continuum_txn`) via
|
|
||||||
:meth:`get_importer_kwargs()`.
|
|
||||||
|
|
||||||
See also :attr:`continuum_uow`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
continuum_comment = None
|
|
||||||
|
|
||||||
def consume_kwargs(self, kwargs):
|
|
||||||
kwargs = super().consume_kwargs(kwargs)
|
|
||||||
self.continuum_comment = kwargs.pop("comment", None)
|
|
||||||
return kwargs
|
|
||||||
|
|
||||||
def begin_target_transaction(self):
|
|
||||||
# pylint: disable=line-too-long
|
|
||||||
"""
|
|
||||||
In addition to normal logic, this does some setup for
|
|
||||||
SQLAlchemy-Continuum:
|
|
||||||
|
|
||||||
It establishes a "unit of work" by calling
|
|
||||||
:meth:`~sqlalchemy-continuum:sqlalchemy_continuum.VersioningManager.unit_of_work()`,
|
|
||||||
assigning the result to :attr:`continuum_uow`.
|
|
||||||
|
|
||||||
It then calls
|
|
||||||
:meth:`~sqlalchemy-continuum:sqlalchemy_continuum.unit_of_work.UnitOfWork.create_transaction()`
|
|
||||||
and assigns that to :attr:`continuum_txn`.
|
|
||||||
|
|
||||||
It also sets the comment for the transaction, if applicable.
|
|
||||||
|
|
||||||
See also docs for parent method:
|
|
||||||
:meth:`~wuttasync.importing.handlers.ToSqlalchemyHandler.begin_target_transaction()`
|
|
||||||
"""
|
|
||||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
|
||||||
|
|
||||||
super().begin_target_transaction()
|
|
||||||
|
|
||||||
self.continuum_uow = continuum.versioning_manager.unit_of_work(
|
|
||||||
self.target_session
|
|
||||||
)
|
|
||||||
|
|
||||||
self.continuum_txn = self.continuum_uow.create_transaction(self.target_session)
|
|
||||||
|
|
||||||
if self.continuum_comment:
|
|
||||||
self.continuum_txn.meta = {"comment": self.continuum_comment}
|
|
||||||
|
|
||||||
def get_importer_kwargs(self, key, **kwargs):
|
|
||||||
"""
|
|
||||||
This modifies the new importer kwargs to add:
|
|
||||||
|
|
||||||
* ``continuum_txn`` - reference to :attr:`continuum_txn`
|
|
||||||
|
|
||||||
See also docs for parent method:
|
|
||||||
:meth:`~wuttasync.importing.handlers.ImportHandler.get_importer_kwargs()`
|
|
||||||
"""
|
|
||||||
kwargs = super().get_importer_kwargs(key, **kwargs)
|
|
||||||
kwargs["continuum_txn"] = self.continuum_txn
|
|
||||||
return kwargs
|
|
||||||
|
|
||||||
# TODO: pylint (correctly) flags this as duplicate code, matching
|
|
||||||
# on the wuttasync.importing.csv module - should fix?
|
|
||||||
def define_importers(self):
|
|
||||||
"""
|
|
||||||
This overrides typical (manual) importer definition, instead
|
|
||||||
generating importers for all versioned models.
|
|
||||||
|
|
||||||
It will inspect the :term:`app model` and call
|
|
||||||
:meth:`make_importer_factory()` for each model found, keeping
|
|
||||||
only the valid importers.
|
|
||||||
|
|
||||||
See also the docs for parent method:
|
|
||||||
:meth:`~wuttasync.importing.handlers.ImportHandler.define_importers()`
|
|
||||||
"""
|
|
||||||
model = self.app.model
|
|
||||||
importers = {}
|
|
||||||
|
|
||||||
# pylint: disable=duplicate-code
|
|
||||||
# mostly try to make an importer for every data model
|
|
||||||
for name in dir(model):
|
|
||||||
cls = getattr(model, name)
|
|
||||||
if (
|
|
||||||
isinstance(cls, type)
|
|
||||||
and issubclass(cls, model.Base)
|
|
||||||
and cls is not model.Base
|
|
||||||
):
|
|
||||||
# only keep "good" importers, i.e. for versioned models
|
|
||||||
if factory := self.make_importer_factory(cls, name):
|
|
||||||
importers[name] = factory
|
|
||||||
|
|
||||||
# sort importers according to schema topography
|
|
||||||
topo_sortkey = make_topo_sortkey(model)
|
|
||||||
importers = OrderedDict(
|
|
||||||
[(name, importers[name]) for name in sorted(importers, key=topo_sortkey)]
|
|
||||||
)
|
|
||||||
|
|
||||||
return importers
|
|
||||||
|
|
||||||
def make_importer_factory(self, model_class, name):
|
|
||||||
"""
|
|
||||||
Try to generate a new :term:`importer` class for the given
|
|
||||||
:term:`data model`. This is called by
|
|
||||||
:meth:`define_importers()`.
|
|
||||||
|
|
||||||
If the provided ``model_class`` is not versioned, this will
|
|
||||||
fail and return ``None``.
|
|
||||||
|
|
||||||
For a versioned model, the new importer class will inherit
|
|
||||||
from :class:`FromWuttaToVersionBase`.
|
|
||||||
|
|
||||||
Its (target)
|
|
||||||
:attr:`~wuttasync.importing.base.Importer.model_class` will be
|
|
||||||
set to the **version** model.
|
|
||||||
|
|
||||||
Its
|
|
||||||
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class`
|
|
||||||
will be set to the **normal** model.
|
|
||||||
|
|
||||||
:param model_class: A (normal, not version) data model class.
|
|
||||||
|
|
||||||
:param name: The "model name" for the importer. New class
|
|
||||||
name will be based on this, so e.g. ``Widget`` model name
|
|
||||||
becomes ``WidgetImporter`` class name.
|
|
||||||
|
|
||||||
:returns: The new class, or ``None``
|
|
||||||
"""
|
|
||||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
|
||||||
|
|
||||||
try:
|
|
||||||
version_class = continuum.version_class(model_class)
|
|
||||||
except continuum.exc.ClassNotVersioned:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return type(
|
|
||||||
f"{name}Importer",
|
|
||||||
(FromWuttaToVersionBase,),
|
|
||||||
{
|
|
||||||
"source_model_class": model_class,
|
|
||||||
"model_class": version_class,
|
|
||||||
"default_keys": list(get_primary_keys(model_class)),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class FromWuttaToVersionBase(FromWuttaMirror, ToWutta):
|
|
||||||
"""
|
|
||||||
Base importer class for Wutta -> Versions.
|
|
||||||
|
|
||||||
This imports from
|
|
||||||
:class:`~wuttasync.importing.wutta.FromWuttaMirror` and
|
|
||||||
:class:`~wuttasync.importing.model.ToWutta`.
|
|
||||||
|
|
||||||
The import handler will dynamically generate importers using this
|
|
||||||
base class; see
|
|
||||||
:meth:`~FromWuttaToVersions.make_importer_factory()`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
continuum_txn = None
|
|
||||||
"""
|
|
||||||
Reference to the handler's attribute of the same name:
|
|
||||||
:attr:`~FromWuttaToVersions.continuum_txn`
|
|
||||||
|
|
||||||
This is the SQLAlchemy-Continuum ``transaction`` record, to which
|
|
||||||
any new version records will associate (if needed).
|
|
||||||
|
|
||||||
This transaction will track the effective user responsible for
|
|
||||||
the change(s), their client IP, and timestamp.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def get_simple_fields(self): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
fields = super().get_simple_fields()
|
|
||||||
unwanted = ["transaction_id", "operation_type", "end_transaction_id"]
|
|
||||||
fields = [field for field in fields if field not in unwanted]
|
|
||||||
return fields
|
|
||||||
|
|
||||||
def get_target_query(self, source_data=None):
|
|
||||||
"""
|
|
||||||
This modifies the normal query to ensure we only get the
|
|
||||||
"latest valid" version for each record, for comparison to
|
|
||||||
source.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
|
|
||||||
In some cases, it still may be possible for multiple
|
|
||||||
"latest" versions to match for a given record. This means
|
|
||||||
inconsistent data; a warning should be logged if so, and
|
|
||||||
you must track it down...
|
|
||||||
|
|
||||||
See also docs for parent method:
|
|
||||||
:meth:`~wuttasync.importing.base.ToSqlalchemy.get_target_query()`
|
|
||||||
"""
|
|
||||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
|
||||||
|
|
||||||
# pylint: disable=singleton-comparison
|
|
||||||
return (
|
|
||||||
self.target_session.query(self.model_class)
|
|
||||||
.filter(self.model_class.end_transaction_id == None)
|
|
||||||
.filter(self.model_class.operation_type != continuum.Operation.DELETE)
|
|
||||||
)
|
|
||||||
|
|
||||||
def normalize_target_object(self, obj): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
data = super().normalize_target_object(obj)
|
|
||||||
|
|
||||||
# we want to add the original version object to normalized
|
|
||||||
# data, so we can access it later for updating if needed. but
|
|
||||||
# this method is called for *both* sides (source+target) since
|
|
||||||
# this is a "mirrored" importer. so we must check the type
|
|
||||||
# and only cache true versions, ignore "normal" objects.
|
|
||||||
if isinstance( # pylint: disable=isinstance-second-argument-not-valid-type
|
|
||||||
obj, self.model_class
|
|
||||||
):
|
|
||||||
data["_objref"] = obj
|
|
||||||
|
|
||||||
return data
|
|
||||||
|
|
||||||
def make_version( # pylint: disable=missing-function-docstring
|
|
||||||
self, source_data, operation_type
|
|
||||||
):
|
|
||||||
key = self.get_record_key(source_data)
|
|
||||||
with self.target_session.no_autoflush:
|
|
||||||
version = self.make_empty_object(key)
|
|
||||||
self.populate(version, source_data)
|
|
||||||
version.transaction = self.continuum_txn
|
|
||||||
version.operation_type = operation_type
|
|
||||||
self.target_session.add(version)
|
|
||||||
return version
|
|
||||||
|
|
||||||
def populate(self, obj, data): # pylint: disable=missing-function-docstring
|
|
||||||
keys = self.get_keys()
|
|
||||||
for field in self.get_simple_fields():
|
|
||||||
if field not in keys and field in data and field in self.fields:
|
|
||||||
setattr(obj, field, data[field])
|
|
||||||
|
|
||||||
def create_target_object(self, key, source_data): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
|
||||||
|
|
||||||
return self.make_version(source_data, continuum.Operation.INSERT)
|
|
||||||
|
|
||||||
def update_target_object( # pylint: disable=empty-docstring
|
|
||||||
self, obj, source_data, target_data=None
|
|
||||||
):
|
|
||||||
""" """
|
|
||||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
|
||||||
|
|
||||||
# when we "update" it always involves making a *new* version
|
|
||||||
# record. but that requires actually updating the "previous"
|
|
||||||
# version to indicate the new version's transaction.
|
|
||||||
prev_version = target_data.pop("_objref")
|
|
||||||
prev_version.end_transaction_id = self.continuum_txn.id
|
|
||||||
|
|
||||||
return self.make_version(source_data, continuum.Operation.UPDATE)
|
|
||||||
|
|
||||||
def delete_target_object(self, obj): # pylint: disable=empty-docstring
|
|
||||||
""" """
|
|
||||||
import sqlalchemy_continuum as continuum # pylint: disable=import-outside-toplevel
|
|
||||||
|
|
||||||
# nb. `obj` here is the existing/old version record; we update
|
|
||||||
# it to indicate the new version's transaction.
|
|
||||||
obj.end_transaction_id = self.continuum_txn.id
|
|
||||||
|
|
||||||
# add new "DELETE" version record. values should be the same as
|
|
||||||
# for "previous" (existing/old) version.
|
|
||||||
source_data = self.normalize_target_object(obj)
|
|
||||||
return self.make_version(source_data, continuum.Operation.DELETE)
|
|
||||||
|
|
@ -21,13 +21,37 @@
|
||||||
#
|
#
|
||||||
################################################################################
|
################################################################################
|
||||||
"""
|
"""
|
||||||
Wutta → Wutta import/export
|
Wutta ⇄ Wutta import/export
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .base import FromSqlalchemyMirror
|
from .handlers import ToSqlalchemyHandler
|
||||||
|
|
||||||
|
|
||||||
class FromWuttaMirror(FromSqlalchemyMirror): # pylint: disable=abstract-method
|
class ToWuttaHandler(ToSqlalchemyHandler):
|
||||||
"""
|
"""
|
||||||
Base class for Wutta -> Wutta data importers.
|
Handler for import/export which targets Wutta ORM (:term:`app
|
||||||
|
database`).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
target_key = "wutta"
|
||||||
|
"" # nb. suppress docs
|
||||||
|
|
||||||
|
def get_target_title(self): # pylint: disable=empty-docstring
|
||||||
|
""" """
|
||||||
|
# nb. we override parent to use app title as default
|
||||||
|
if hasattr(self, "target_title"):
|
||||||
|
return self.target_title
|
||||||
|
if hasattr(self, "generic_target_title"):
|
||||||
|
return self.generic_target_title
|
||||||
|
return self.app.get_title()
|
||||||
|
|
||||||
|
def make_target_session(self):
|
||||||
|
"""
|
||||||
|
Call
|
||||||
|
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
||||||
|
and return the result.
|
||||||
|
|
||||||
|
:returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
|
||||||
|
instance.
|
||||||
|
"""
|
||||||
|
return self.app.make_session()
|
||||||
|
|
|
||||||
|
|
@ -1,68 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
################################################################################
|
|
||||||
#
|
|
||||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
|
||||||
# Copyright © 2024-2025 Lance Edgar
|
|
||||||
#
|
|
||||||
# This file is part of Wutta Framework.
|
|
||||||
#
|
|
||||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
|
||||||
# under the terms of the GNU General Public License as published by the Free
|
|
||||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
|
||||||
# later version.
|
|
||||||
#
|
|
||||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
|
||||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
||||||
# more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License along with
|
|
||||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
#
|
|
||||||
################################################################################
|
|
||||||
"""
|
|
||||||
Testing utilities
|
|
||||||
"""
|
|
||||||
|
|
||||||
from wuttjamaican.testing import ConfigTestCase
|
|
||||||
|
|
||||||
|
|
||||||
class ImportExportWarningTestCase(ConfigTestCase):
|
|
||||||
"""
|
|
||||||
Base class for testing the import/export warning email settings.
|
|
||||||
|
|
||||||
This inherits from
|
|
||||||
:class:`~wuttjamaican:wuttjamaican.testing.ConfigTestCase`.
|
|
||||||
|
|
||||||
Example usage::
|
|
||||||
|
|
||||||
from wuttasync.testing import ImportExportWarningTestCase
|
|
||||||
|
|
||||||
class TestEmailSettings(ImportExportWarningTestCase):
|
|
||||||
|
|
||||||
def test_import_to_wutta_from_foo_warning(self):
|
|
||||||
self.do_test_preview("import_to_wutta_from_foo_warning")
|
|
||||||
|
|
||||||
def test_export_to_foo_from_wutta_warning(self):
|
|
||||||
self.do_test_preview("export_to_foo_from_wutta_warning")
|
|
||||||
"""
|
|
||||||
|
|
||||||
app_title = "Wutta Poser"
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
self.setup_config()
|
|
||||||
self.config.setdefault("wutta.app_title", self.app_title)
|
|
||||||
|
|
||||||
def make_preview( # pylint: disable=missing-function-docstring,unused-argument
|
|
||||||
self, key, mode="html"
|
|
||||||
):
|
|
||||||
handler = self.app.get_email_handler()
|
|
||||||
setting = handler.get_email_setting(key)
|
|
||||||
context = setting.sample_data()
|
|
||||||
return handler.get_auto_html_body(
|
|
||||||
setting.key, context, fallback_key=setting.fallback_key
|
|
||||||
)
|
|
||||||
|
|
||||||
def do_test_preview(self, key): # pylint: disable=missing-function-docstring
|
|
||||||
body = self.make_preview(key, mode="html")
|
|
||||||
self.assertIn("Diff warning for ", body)
|
|
||||||
|
|
@ -34,10 +34,6 @@ class TestImportCommandHandler(DataTestCase):
|
||||||
handler = self.make_handler(import_handler=myhandler)
|
handler = self.make_handler(import_handler=myhandler)
|
||||||
self.assertIs(handler.import_handler, myhandler)
|
self.assertIs(handler.import_handler, myhandler)
|
||||||
|
|
||||||
# as key
|
|
||||||
handler = self.make_handler(key="import.to_wutta.from_csv")
|
|
||||||
self.assertIsInstance(handler.import_handler, FromCsvToWutta)
|
|
||||||
|
|
||||||
def test_run(self):
|
def test_run(self):
|
||||||
handler = self.make_handler(
|
handler = self.make_handler(
|
||||||
import_handler="wuttasync.importing.csv:FromCsvToWutta"
|
import_handler="wuttasync.importing.csv:FromCsvToWutta"
|
||||||
|
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
|
|
||||||
from unittest import TestCase
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
|
|
||||||
from wuttasync.cli import import_versions as mod, ImportCommandHandler
|
|
||||||
|
|
||||||
|
|
||||||
class TestImportCsv(TestCase):
|
|
||||||
|
|
||||||
def test_basic(self):
|
|
||||||
params = {
|
|
||||||
"models": [],
|
|
||||||
"create": True,
|
|
||||||
"update": True,
|
|
||||||
"delete": False,
|
|
||||||
"dry_run": True,
|
|
||||||
}
|
|
||||||
ctx = MagicMock(params=params)
|
|
||||||
with patch.object(ImportCommandHandler, "run") as run:
|
|
||||||
mod.import_versions(ctx)
|
|
||||||
run.assert_called_once_with(params)
|
|
||||||
|
|
@ -2,8 +2,6 @@
|
||||||
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from sqlalchemy import orm
|
|
||||||
|
|
||||||
from wuttjamaican.testing import DataTestCase
|
from wuttjamaican.testing import DataTestCase
|
||||||
|
|
||||||
from wuttasync.importing import base as mod, ImportHandler, Orientation
|
from wuttasync.importing import base as mod, ImportHandler, Orientation
|
||||||
|
|
@ -80,31 +78,13 @@ class TestImporter(DataTestCase):
|
||||||
|
|
||||||
def test_get_keys(self):
|
def test_get_keys(self):
|
||||||
model = self.app.model
|
model = self.app.model
|
||||||
|
|
||||||
# nb. get_keys() will cache the return value, so must
|
|
||||||
# re-create importer for each test
|
|
||||||
|
|
||||||
# keys inspected from model by default
|
|
||||||
imp = self.make_importer(model_class=model.Setting)
|
imp = self.make_importer(model_class=model.Setting)
|
||||||
self.assertEqual(imp.get_keys(), ["name"])
|
self.assertEqual(imp.get_keys(), ["name"])
|
||||||
imp = self.make_importer(model_class=model.User)
|
with patch.multiple(imp, create=True, key="value"):
|
||||||
self.assertEqual(imp.get_keys(), ["uuid"])
|
self.assertEqual(imp.get_keys(), ["value"])
|
||||||
|
with patch.multiple(imp, create=True, keys=["foo", "bar"]):
|
||||||
# class may define 'keys'
|
|
||||||
imp = self.make_importer(model_class=model.User)
|
|
||||||
with patch.object(imp, "keys", new=["foo", "bar"], create=True):
|
|
||||||
self.assertEqual(imp.get_keys(), ["foo", "bar"])
|
self.assertEqual(imp.get_keys(), ["foo", "bar"])
|
||||||
|
|
||||||
# class may define 'key'
|
|
||||||
imp = self.make_importer(model_class=model.User)
|
|
||||||
with patch.object(imp, "key", new="whatever", create=True):
|
|
||||||
self.assertEqual(imp.get_keys(), ["whatever"])
|
|
||||||
|
|
||||||
# class may define 'default_keys'
|
|
||||||
imp = self.make_importer(model_class=model.User)
|
|
||||||
with patch.object(imp, "default_keys", new=["baz", "foo"]):
|
|
||||||
self.assertEqual(imp.get_keys(), ["baz", "foo"])
|
|
||||||
|
|
||||||
def test_process_data(self):
|
def test_process_data(self):
|
||||||
model = self.app.model
|
model = self.app.model
|
||||||
imp = self.make_importer(
|
imp = self.make_importer(
|
||||||
|
|
@ -671,106 +651,6 @@ class TestFromFile(DataTestCase):
|
||||||
close.assert_called_once_with()
|
close.assert_called_once_with()
|
||||||
|
|
||||||
|
|
||||||
class TestQueryWrapper(DataTestCase):
|
|
||||||
|
|
||||||
def test_basic(self):
|
|
||||||
model = self.app.model
|
|
||||||
|
|
||||||
p1 = model.Person(full_name="John Doe")
|
|
||||||
self.session.add(p1)
|
|
||||||
p2 = model.Person(full_name="Jane Doe")
|
|
||||||
self.session.add(p2)
|
|
||||||
self.session.commit()
|
|
||||||
|
|
||||||
# cannot get count via len(query), must use query.count()
|
|
||||||
query = self.session.query(model.Person)
|
|
||||||
self.assertEqual(query.count(), 2)
|
|
||||||
self.assertRaises(TypeError, len, query)
|
|
||||||
|
|
||||||
# but can use len(wrapper)
|
|
||||||
wrapper = mod.QueryWrapper(query)
|
|
||||||
self.assertEqual(len(wrapper), 2)
|
|
||||||
|
|
||||||
# iter(wrapper) should work too
|
|
||||||
people = [p for p in wrapper]
|
|
||||||
self.assertEqual(people, [p1, p2])
|
|
||||||
people = [p for p in iter(wrapper)]
|
|
||||||
self.assertEqual(people, [p1, p2])
|
|
||||||
people = [p for p in list(wrapper)]
|
|
||||||
self.assertEqual(people, [p1, p2])
|
|
||||||
|
|
||||||
|
|
||||||
class TestFromSqlalchemy(DataTestCase):
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
self.setup_db()
|
|
||||||
self.handler = ImportHandler(self.config)
|
|
||||||
|
|
||||||
def make_importer(self, **kwargs):
|
|
||||||
kwargs.setdefault("handler", self.handler)
|
|
||||||
return mod.FromSqlalchemy(self.config, **kwargs)
|
|
||||||
|
|
||||||
def test_get_source_query(self):
|
|
||||||
model = self.app.model
|
|
||||||
imp = self.make_importer(
|
|
||||||
source_model_class=model.Upgrade, source_session=self.session
|
|
||||||
)
|
|
||||||
query = imp.get_source_query()
|
|
||||||
self.assertIsInstance(query, orm.Query)
|
|
||||||
froms = query.selectable.get_final_froms()
|
|
||||||
self.assertEqual(len(froms), 1)
|
|
||||||
table = froms[0]
|
|
||||||
self.assertEqual(table.name, "upgrade")
|
|
||||||
|
|
||||||
def test_get_source_objects(self):
|
|
||||||
model = self.app.model
|
|
||||||
|
|
||||||
user1 = model.User(username="fred")
|
|
||||||
self.session.add(user1)
|
|
||||||
user2 = model.User(username="bettie")
|
|
||||||
self.session.add(user2)
|
|
||||||
self.session.commit()
|
|
||||||
|
|
||||||
imp = self.make_importer(
|
|
||||||
source_model_class=model.User, source_session=self.session
|
|
||||||
)
|
|
||||||
result = imp.get_source_objects()
|
|
||||||
self.assertIsInstance(result, mod.QueryWrapper)
|
|
||||||
self.assertEqual(len(result), 2)
|
|
||||||
self.assertEqual(list(result), [user1, user2])
|
|
||||||
|
|
||||||
|
|
||||||
class TestFromSqlalchemyMirror(DataTestCase):
|
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
self.setup_db()
|
|
||||||
self.handler = ImportHandler(self.config)
|
|
||||||
|
|
||||||
def make_importer(self, **kwargs):
|
|
||||||
kwargs.setdefault("handler", self.handler)
|
|
||||||
return mod.FromSqlalchemyMirror(self.config, **kwargs)
|
|
||||||
|
|
||||||
def test_source_model_class(self):
|
|
||||||
model = self.app.model
|
|
||||||
|
|
||||||
# source_model_class will mirror model_class
|
|
||||||
imp = self.make_importer(model_class=model.Upgrade)
|
|
||||||
self.assertIs(imp.model_class, model.Upgrade)
|
|
||||||
self.assertIs(imp.source_model_class, model.Upgrade)
|
|
||||||
|
|
||||||
def test_normalize_source_object(self):
|
|
||||||
model = self.app.model
|
|
||||||
imp = self.make_importer(model_class=model.Upgrade)
|
|
||||||
upgrade = model.Upgrade()
|
|
||||||
|
|
||||||
# normalize_source_object() should invoke normalize_target_object()
|
|
||||||
with patch.object(imp, "normalize_target_object") as normalize_target_object:
|
|
||||||
normalize_target_object.return_value = 42
|
|
||||||
result = imp.normalize_source_object(upgrade)
|
|
||||||
self.assertEqual(result, 42)
|
|
||||||
normalize_target_object.assert_called_once_with(upgrade)
|
|
||||||
|
|
||||||
|
|
||||||
class TestToSqlalchemy(DataTestCase):
|
class TestToSqlalchemy(DataTestCase):
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
|
|
||||||
|
|
@ -2,18 +2,12 @@
|
||||||
|
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
from uuid import UUID
|
|
||||||
|
|
||||||
from wuttjamaican.testing import DataTestCase
|
from wuttjamaican.testing import DataTestCase
|
||||||
|
|
||||||
from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy
|
from wuttasync.importing import handlers as mod, Importer, ToSqlalchemy
|
||||||
|
|
||||||
|
|
||||||
class FromFooToBar(mod.ImportHandler):
|
|
||||||
source_key = "foo"
|
|
||||||
target_key = "bar"
|
|
||||||
|
|
||||||
|
|
||||||
class TestImportHandler(DataTestCase):
|
class TestImportHandler(DataTestCase):
|
||||||
|
|
||||||
def make_handler(self, **kwargs):
|
def make_handler(self, **kwargs):
|
||||||
|
|
@ -36,10 +30,10 @@ class TestImportHandler(DataTestCase):
|
||||||
|
|
||||||
def test_get_key(self):
|
def test_get_key(self):
|
||||||
handler = self.make_handler()
|
handler = self.make_handler()
|
||||||
self.assertEqual(handler.get_key(), "import.to_None.from_None")
|
self.assertEqual(handler.get_key(), "to_None.from_None.import")
|
||||||
|
|
||||||
with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"):
|
with patch.multiple(mod.ImportHandler, source_key="csv", target_key="wutta"):
|
||||||
self.assertEqual(handler.get_key(), "import.to_wutta.from_csv")
|
self.assertEqual(handler.get_key(), "to_wutta.from_csv.import")
|
||||||
|
|
||||||
def test_get_spec(self):
|
def test_get_spec(self):
|
||||||
handler = self.make_handler()
|
handler = self.make_handler()
|
||||||
|
|
@ -155,41 +149,15 @@ class TestImportHandler(DataTestCase):
|
||||||
kw = {}
|
kw = {}
|
||||||
result = handler.consume_kwargs(kw)
|
result = handler.consume_kwargs(kw)
|
||||||
self.assertIs(result, kw)
|
self.assertIs(result, kw)
|
||||||
self.assertEqual(result, {})
|
|
||||||
|
|
||||||
# dry_run (not consumed)
|
# captures dry-run flag
|
||||||
self.assertFalse(handler.dry_run)
|
self.assertFalse(handler.dry_run)
|
||||||
kw["dry_run"] = True
|
kw["dry_run"] = True
|
||||||
result = handler.consume_kwargs(kw)
|
result = handler.consume_kwargs(kw)
|
||||||
self.assertIs(result, kw)
|
self.assertIs(result, kw)
|
||||||
self.assertIn("dry_run", kw)
|
|
||||||
self.assertTrue(kw["dry_run"])
|
self.assertTrue(kw["dry_run"])
|
||||||
self.assertTrue(handler.dry_run)
|
self.assertTrue(handler.dry_run)
|
||||||
|
|
||||||
# warnings (consumed)
|
|
||||||
self.assertFalse(handler.warnings)
|
|
||||||
kw["warnings"] = True
|
|
||||||
result = handler.consume_kwargs(kw)
|
|
||||||
self.assertIs(result, kw)
|
|
||||||
self.assertNotIn("warnings", kw)
|
|
||||||
self.assertTrue(handler.warnings)
|
|
||||||
|
|
||||||
# warnings_recipients (consumed)
|
|
||||||
self.assertIsNone(handler.warnings_recipients)
|
|
||||||
kw["warnings_recipients"] = "bob@example.com"
|
|
||||||
result = handler.consume_kwargs(kw)
|
|
||||||
self.assertIs(result, kw)
|
|
||||||
self.assertNotIn("warnings_recipients", kw)
|
|
||||||
self.assertEqual(handler.warnings_recipients, ["bob@example.com"])
|
|
||||||
|
|
||||||
# warnings_max_diffs (consumed)
|
|
||||||
self.assertEqual(handler.warnings_max_diffs, 15)
|
|
||||||
kw["warnings_max_diffs"] = 30
|
|
||||||
result = handler.consume_kwargs(kw)
|
|
||||||
self.assertIs(result, kw)
|
|
||||||
self.assertNotIn("warnings_max_diffs", kw)
|
|
||||||
self.assertEqual(handler.warnings_max_diffs, 30)
|
|
||||||
|
|
||||||
def test_define_importers(self):
|
def test_define_importers(self):
|
||||||
handler = self.make_handler()
|
handler = self.make_handler()
|
||||||
importers = handler.define_importers()
|
importers = handler.define_importers()
|
||||||
|
|
@ -219,94 +187,6 @@ class TestImportHandler(DataTestCase):
|
||||||
KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting
|
KeyError, handler.get_importer, "BunchOfNonsense", model_class=model.Setting
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_get_warnings_email_key(self):
|
|
||||||
handler = FromFooToBar(self.config)
|
|
||||||
|
|
||||||
# default
|
|
||||||
key = handler.get_warnings_email_key()
|
|
||||||
self.assertEqual(key, "import_to_bar_from_foo_warning")
|
|
||||||
|
|
||||||
# override
|
|
||||||
handler.warnings_email_key = "from_foo_to_bar"
|
|
||||||
key = handler.get_warnings_email_key()
|
|
||||||
self.assertEqual(key, "from_foo_to_bar")
|
|
||||||
|
|
||||||
def test_process_changes(self):
|
|
||||||
model = self.app.model
|
|
||||||
handler = self.make_handler()
|
|
||||||
email_handler = self.app.get_email_handler()
|
|
||||||
|
|
||||||
handler.process_started = self.app.localtime()
|
|
||||||
|
|
||||||
alice = model.User(username="alice")
|
|
||||||
bob = model.User(username="bob")
|
|
||||||
charlie = model.User(username="charlie")
|
|
||||||
|
|
||||||
changes = {
|
|
||||||
"User": (
|
|
||||||
[
|
|
||||||
(
|
|
||||||
alice,
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-79db-8000-ce40345044fe"),
|
|
||||||
"username": "alice",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
(
|
|
||||||
bob,
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
|
||||||
"username": "bob",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-7a8c-8000-05d78792b084"),
|
|
||||||
"username": "bobbie",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
(
|
|
||||||
charlie,
|
|
||||||
{
|
|
||||||
"uuid": UUID("06946d64-1ebf-7ad4-8000-1ba52f720c48"),
|
|
||||||
"username": "charlie",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
],
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
# no email if not in warnings mode
|
|
||||||
self.assertFalse(handler.warnings)
|
|
||||||
with patch.object(self.app, "send_email") as send_email:
|
|
||||||
handler.process_changes(changes)
|
|
||||||
send_email.assert_not_called()
|
|
||||||
|
|
||||||
# email sent (to default recip) if in warnings mode
|
|
||||||
handler.warnings = True
|
|
||||||
self.config.setdefault("wutta.email.default.to", "admin@example.com")
|
|
||||||
with patch.object(email_handler, "deliver_message") as deliver_message:
|
|
||||||
handler.process_changes(changes)
|
|
||||||
deliver_message.assert_called_once()
|
|
||||||
args, kwargs = deliver_message.call_args
|
|
||||||
self.assertEqual(kwargs, {"recips": None})
|
|
||||||
self.assertEqual(len(args), 1)
|
|
||||||
msg = args[0]
|
|
||||||
self.assertEqual(msg.to, ["admin@example.com"])
|
|
||||||
|
|
||||||
# can override email recip
|
|
||||||
handler.warnings_recipients = ["bob@example.com"]
|
|
||||||
with patch.object(email_handler, "deliver_message") as deliver_message:
|
|
||||||
handler.process_changes(changes)
|
|
||||||
deliver_message.assert_called_once()
|
|
||||||
args, kwargs = deliver_message.call_args
|
|
||||||
self.assertEqual(kwargs, {"recips": None})
|
|
||||||
self.assertEqual(len(args), 1)
|
|
||||||
msg = args[0]
|
|
||||||
self.assertEqual(msg.to, ["bob@example.com"])
|
|
||||||
|
|
||||||
|
|
||||||
class TestFromFileHandler(DataTestCase):
|
class TestFromFileHandler(DataTestCase):
|
||||||
|
|
||||||
|
|
@ -333,97 +213,6 @@ class TestFromFileHandler(DataTestCase):
|
||||||
process_data.assert_called_once_with(input_file_dir=self.tempdir)
|
process_data.assert_called_once_with(input_file_dir=self.tempdir)
|
||||||
|
|
||||||
|
|
||||||
class TestFromSqlalchemyHandler(DataTestCase):
|
|
||||||
|
|
||||||
def make_handler(self, **kwargs):
|
|
||||||
return mod.FromSqlalchemyHandler(self.config, **kwargs)
|
|
||||||
|
|
||||||
def test_make_source_session(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
self.assertRaises(NotImplementedError, handler.make_source_session)
|
|
||||||
|
|
||||||
def test_begin_source_transaction(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
self.assertIsNone(handler.source_session)
|
|
||||||
with patch.object(handler, "make_source_session", return_value=self.session):
|
|
||||||
handler.begin_source_transaction()
|
|
||||||
self.assertIs(handler.source_session, self.session)
|
|
||||||
|
|
||||||
def test_commit_source_transaction(self):
|
|
||||||
model = self.app.model
|
|
||||||
handler = self.make_handler()
|
|
||||||
handler.source_session = self.session
|
|
||||||
self.assertEqual(self.session.query(model.User).count(), 0)
|
|
||||||
|
|
||||||
# nb. do not commit this yet
|
|
||||||
user = model.User(username="fred")
|
|
||||||
self.session.add(user)
|
|
||||||
|
|
||||||
self.assertTrue(self.session.in_transaction())
|
|
||||||
self.assertIn(user, self.session)
|
|
||||||
handler.commit_source_transaction()
|
|
||||||
self.assertIsNone(handler.source_session)
|
|
||||||
self.assertFalse(self.session.in_transaction())
|
|
||||||
self.assertNotIn(user, self.session) # hm, surprising?
|
|
||||||
self.assertEqual(self.session.query(model.User).count(), 1)
|
|
||||||
|
|
||||||
def test_rollback_source_transaction(self):
|
|
||||||
model = self.app.model
|
|
||||||
handler = self.make_handler()
|
|
||||||
handler.source_session = self.session
|
|
||||||
self.assertEqual(self.session.query(model.User).count(), 0)
|
|
||||||
|
|
||||||
# nb. do not commit this yet
|
|
||||||
user = model.User(username="fred")
|
|
||||||
self.session.add(user)
|
|
||||||
|
|
||||||
self.assertTrue(self.session.in_transaction())
|
|
||||||
self.assertIn(user, self.session)
|
|
||||||
handler.rollback_source_transaction()
|
|
||||||
self.assertIsNone(handler.source_session)
|
|
||||||
self.assertFalse(self.session.in_transaction())
|
|
||||||
self.assertNotIn(user, self.session)
|
|
||||||
self.assertEqual(self.session.query(model.User).count(), 0)
|
|
||||||
|
|
||||||
def test_get_importer_kwargs(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
handler.source_session = self.session
|
|
||||||
kw = handler.get_importer_kwargs("User")
|
|
||||||
self.assertIn("source_session", kw)
|
|
||||||
self.assertIs(kw["source_session"], self.session)
|
|
||||||
|
|
||||||
|
|
||||||
class TestFromWuttaHandler(DataTestCase):
|
|
||||||
|
|
||||||
def make_handler(self, **kwargs):
|
|
||||||
return mod.FromWuttaHandler(self.config, **kwargs)
|
|
||||||
|
|
||||||
def test_get_source_title(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
|
|
||||||
# uses app title by default
|
|
||||||
self.config.setdefault("wutta.app_title", "What About This")
|
|
||||||
self.assertEqual(handler.get_source_title(), "What About This")
|
|
||||||
|
|
||||||
# or generic default if present
|
|
||||||
handler.generic_source_title = "WHATABOUTTHIS"
|
|
||||||
self.assertEqual(handler.get_source_title(), "WHATABOUTTHIS")
|
|
||||||
|
|
||||||
# but prefer specific title if present
|
|
||||||
handler.source_title = "what_about_this"
|
|
||||||
self.assertEqual(handler.get_source_title(), "what_about_this")
|
|
||||||
|
|
||||||
def test_make_source_session(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
|
|
||||||
# makes "new" (mocked in our case) app session
|
|
||||||
with patch.object(self.app, "make_session") as make_session:
|
|
||||||
make_session.return_value = self.session
|
|
||||||
session = handler.make_source_session()
|
|
||||||
make_session.assert_called_once_with()
|
|
||||||
self.assertIs(session, self.session)
|
|
||||||
|
|
||||||
|
|
||||||
class TestToSqlalchemyHandler(DataTestCase):
|
class TestToSqlalchemyHandler(DataTestCase):
|
||||||
|
|
||||||
def make_handler(self, **kwargs):
|
def make_handler(self, **kwargs):
|
||||||
|
|
@ -467,34 +256,3 @@ class TestToSqlalchemyHandler(DataTestCase):
|
||||||
kw = handler.get_importer_kwargs("Setting")
|
kw = handler.get_importer_kwargs("Setting")
|
||||||
self.assertIn("target_session", kw)
|
self.assertIn("target_session", kw)
|
||||||
self.assertIs(kw["target_session"], self.session)
|
self.assertIs(kw["target_session"], self.session)
|
||||||
|
|
||||||
|
|
||||||
class TestToWuttaHandler(DataTestCase):
|
|
||||||
|
|
||||||
def make_handler(self, **kwargs):
|
|
||||||
return mod.ToWuttaHandler(self.config, **kwargs)
|
|
||||||
|
|
||||||
def test_get_target_title(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
|
|
||||||
# uses app title by default
|
|
||||||
self.config.setdefault("wutta.app_title", "What About This")
|
|
||||||
self.assertEqual(handler.get_target_title(), "What About This")
|
|
||||||
|
|
||||||
# or generic default if present
|
|
||||||
handler.generic_target_title = "WHATABOUTTHIS"
|
|
||||||
self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS")
|
|
||||||
|
|
||||||
# but prefer specific title if present
|
|
||||||
handler.target_title = "what_about_this"
|
|
||||||
self.assertEqual(handler.get_target_title(), "what_about_this")
|
|
||||||
|
|
||||||
def test_make_target_session(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
|
|
||||||
# makes "new" (mocked in our case) app session
|
|
||||||
with patch.object(self.app, "make_session") as make_session:
|
|
||||||
make_session.return_value = self.session
|
|
||||||
session = handler.make_target_session()
|
|
||||||
make_session.assert_called_once_with()
|
|
||||||
self.assertIs(session, self.session)
|
|
||||||
|
|
|
||||||
|
|
@ -1,262 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
|
|
||||||
from sqlalchemy import orm
|
|
||||||
import sqlalchemy_continuum as continuum
|
|
||||||
|
|
||||||
from wuttjamaican.util import make_true_uuid
|
|
||||||
from wutta_continuum.testing import VersionTestCase
|
|
||||||
|
|
||||||
from wuttasync.importing import versions as mod, Importer
|
|
||||||
|
|
||||||
|
|
||||||
class TestFromWuttaToVersions(VersionTestCase):
|
|
||||||
|
|
||||||
def make_handler(self, **kwargs):
|
|
||||||
return mod.FromWuttaToVersions(self.config, **kwargs)
|
|
||||||
|
|
||||||
def test_consume_kwargs(self):
|
|
||||||
|
|
||||||
# no comment by default
|
|
||||||
handler = self.make_handler()
|
|
||||||
kw = handler.consume_kwargs({})
|
|
||||||
self.assertEqual(kw, {})
|
|
||||||
self.assertIsNone(handler.continuum_comment)
|
|
||||||
|
|
||||||
# but can provide one
|
|
||||||
handler = self.make_handler()
|
|
||||||
kw = handler.consume_kwargs({"comment": "yeehaw"})
|
|
||||||
self.assertEqual(kw, {})
|
|
||||||
self.assertEqual(handler.continuum_comment, "yeehaw")
|
|
||||||
|
|
||||||
def test_begin_target_transaction(self):
|
|
||||||
model = self.app.model
|
|
||||||
txncls = continuum.transaction_class(model.User)
|
|
||||||
|
|
||||||
# basic / defaults
|
|
||||||
handler = self.make_handler()
|
|
||||||
self.assertIsNone(handler.continuum_uow)
|
|
||||||
self.assertIsNone(handler.continuum_txn)
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
self.assertIsInstance(handler.continuum_uow, continuum.UnitOfWork)
|
|
||||||
self.assertIsInstance(handler.continuum_txn, txncls)
|
|
||||||
# nb. no comment
|
|
||||||
self.assertIsNone(handler.continuum_txn.meta.get("comment"))
|
|
||||||
|
|
||||||
# with comment
|
|
||||||
handler = self.make_handler()
|
|
||||||
handler.continuum_comment = "yeehaw"
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
self.assertIn("comment", handler.continuum_txn.meta)
|
|
||||||
self.assertEqual(handler.continuum_txn.meta["comment"], "yeehaw")
|
|
||||||
|
|
||||||
def test_get_importer_kwargs(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
|
|
||||||
kw = handler.get_importer_kwargs("User")
|
|
||||||
self.assertIn("continuum_txn", kw)
|
|
||||||
self.assertIs(kw["continuum_txn"], handler.continuum_txn)
|
|
||||||
|
|
||||||
def test_make_importer_factory(self):
|
|
||||||
model = self.app.model
|
|
||||||
handler = self.make_handler()
|
|
||||||
|
|
||||||
# versioned class
|
|
||||||
factory = handler.make_importer_factory(model.User, "User")
|
|
||||||
self.assertTrue(issubclass(factory, mod.FromWuttaToVersionBase))
|
|
||||||
self.assertIs(factory.source_model_class, model.User)
|
|
||||||
self.assertIs(factory.model_class, continuum.version_class(model.User))
|
|
||||||
|
|
||||||
# non-versioned
|
|
||||||
factory = handler.make_importer_factory(model.Upgrade, "Upgrade")
|
|
||||||
self.assertIsNone(factory)
|
|
||||||
|
|
||||||
def test_define_importers(self):
|
|
||||||
handler = self.make_handler()
|
|
||||||
|
|
||||||
importers = handler.define_importers()
|
|
||||||
self.assertIn("User", importers)
|
|
||||||
self.assertIn("Person", importers)
|
|
||||||
self.assertNotIn("Upgrade", importers)
|
|
||||||
|
|
||||||
|
|
||||||
class TestFromWuttaToVersionBase(VersionTestCase):
|
|
||||||
|
|
||||||
def make_importer(self, model_class=None, **kwargs):
|
|
||||||
imp = mod.FromWuttaToVersionBase(self.config, **kwargs)
|
|
||||||
if model_class:
|
|
||||||
imp.model_class = model_class
|
|
||||||
return imp
|
|
||||||
|
|
||||||
def test_get_simple_fields(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
|
|
||||||
# first confirm what a "normal" importer would do
|
|
||||||
imp = Importer(self.config, model_class=vercls)
|
|
||||||
fields = imp.get_simple_fields()
|
|
||||||
self.assertIn("username", fields)
|
|
||||||
self.assertIn("person_uuid", fields)
|
|
||||||
self.assertIn("transaction_id", fields)
|
|
||||||
self.assertIn("operation_type", fields)
|
|
||||||
self.assertIn("end_transaction_id", fields)
|
|
||||||
|
|
||||||
# now test what the "version" importer does
|
|
||||||
imp = self.make_importer(model_class=vercls)
|
|
||||||
fields = imp.get_simple_fields()
|
|
||||||
self.assertIn("username", fields)
|
|
||||||
self.assertIn("person_uuid", fields)
|
|
||||||
self.assertNotIn("transaction_id", fields)
|
|
||||||
self.assertNotIn("operation_type", fields)
|
|
||||||
self.assertNotIn("end_transaction_id", fields)
|
|
||||||
|
|
||||||
def test_get_target_query(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
imp = self.make_importer(model_class=vercls, target_session=self.session)
|
|
||||||
|
|
||||||
# TODO: not sure what else to test here..
|
|
||||||
query = imp.get_target_query()
|
|
||||||
self.assertIsInstance(query, orm.Query)
|
|
||||||
|
|
||||||
def test_normalize_target_object(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
imp = self.make_importer(model_class=vercls)
|
|
||||||
|
|
||||||
user = model.User(username="fred")
|
|
||||||
self.session.add(user)
|
|
||||||
self.session.commit()
|
|
||||||
version = user.versions[0]
|
|
||||||
|
|
||||||
# version object should be embedded in data dict
|
|
||||||
data = imp.normalize_target_object(version)
|
|
||||||
self.assertIsInstance(data, dict)
|
|
||||||
self.assertIn("_objref", data)
|
|
||||||
self.assertIs(data["_objref"], version)
|
|
||||||
|
|
||||||
# but normal object is not embedded
|
|
||||||
data = imp.normalize_target_object(user)
|
|
||||||
self.assertIsInstance(data, dict)
|
|
||||||
self.assertNotIn("_version", data)
|
|
||||||
|
|
||||||
def test_make_version(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
|
|
||||||
user = model.User(username="fred")
|
|
||||||
self.session.add(user)
|
|
||||||
self.session.commit()
|
|
||||||
|
|
||||||
handler = mod.FromWuttaToVersions(self.config)
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
handler.target_session.close()
|
|
||||||
handler.target_session = self.session
|
|
||||||
|
|
||||||
imp = self.make_importer(
|
|
||||||
model_class=vercls,
|
|
||||||
fields=["uuid", "username"],
|
|
||||||
keys=("uuid",),
|
|
||||||
target_session=self.session,
|
|
||||||
continuum_txn=handler.continuum_txn,
|
|
||||||
)
|
|
||||||
|
|
||||||
data = {"uuid": user.uuid, "username": "freddie"}
|
|
||||||
version = imp.make_version(data, continuum.Operation.UPDATE)
|
|
||||||
self.assertIsInstance(version, vercls)
|
|
||||||
self.assertEqual(version.uuid, user.uuid)
|
|
||||||
self.assertEqual(version.username, "freddie")
|
|
||||||
self.assertIn(version, self.session)
|
|
||||||
self.assertIs(version.transaction, imp.continuum_txn)
|
|
||||||
self.assertEqual(version.operation_type, continuum.Operation.UPDATE)
|
|
||||||
|
|
||||||
def test_create_target_object(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
|
|
||||||
handler = mod.FromWuttaToVersions(self.config)
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
handler.target_session.close()
|
|
||||||
handler.target_session = self.session
|
|
||||||
|
|
||||||
imp = self.make_importer(
|
|
||||||
model_class=vercls,
|
|
||||||
fields=["uuid", "username"],
|
|
||||||
keys=("uuid",),
|
|
||||||
target_session=self.session,
|
|
||||||
continuum_txn=handler.continuum_txn,
|
|
||||||
)
|
|
||||||
|
|
||||||
source_data = {"uuid": make_true_uuid(), "username": "bettie"}
|
|
||||||
self.assertEqual(self.session.query(vercls).count(), 0)
|
|
||||||
version = imp.create_target_object((source_data["uuid"], 1), source_data)
|
|
||||||
self.assertEqual(self.session.query(vercls).count(), 1)
|
|
||||||
self.assertEqual(version.transaction_id, imp.continuum_txn.id)
|
|
||||||
self.assertEqual(version.operation_type, continuum.Operation.INSERT)
|
|
||||||
self.assertIsNone(version.end_transaction_id)
|
|
||||||
|
|
||||||
def test_update_target_object(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
|
|
||||||
user = model.User(username="fred")
|
|
||||||
self.session.add(user)
|
|
||||||
self.session.commit()
|
|
||||||
version1 = user.versions[0]
|
|
||||||
|
|
||||||
handler = mod.FromWuttaToVersions(self.config)
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
handler.target_session.close()
|
|
||||||
handler.target_session = self.session
|
|
||||||
|
|
||||||
imp = self.make_importer(
|
|
||||||
model_class=vercls,
|
|
||||||
fields=["uuid", "username"],
|
|
||||||
keys=("uuid",),
|
|
||||||
target_session=self.session,
|
|
||||||
continuum_txn=handler.continuum_txn,
|
|
||||||
)
|
|
||||||
|
|
||||||
source_data = {"uuid": user.uuid, "username": "freddie"}
|
|
||||||
target_data = imp.normalize_target_object(version1)
|
|
||||||
self.assertEqual(self.session.query(vercls).count(), 1)
|
|
||||||
self.assertIsNone(version1.end_transaction_id)
|
|
||||||
version2 = imp.update_target_object(
|
|
||||||
version1, source_data, target_data=target_data
|
|
||||||
)
|
|
||||||
self.assertEqual(self.session.query(vercls).count(), 2)
|
|
||||||
self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id)
|
|
||||||
self.assertEqual(version2.transaction_id, imp.continuum_txn.id)
|
|
||||||
self.assertEqual(version2.operation_type, continuum.Operation.UPDATE)
|
|
||||||
self.assertIsNone(version2.end_transaction_id)
|
|
||||||
|
|
||||||
def test_delete_target_object(self):
|
|
||||||
model = self.app.model
|
|
||||||
vercls = continuum.version_class(model.User)
|
|
||||||
|
|
||||||
user = model.User(username="fred")
|
|
||||||
self.session.add(user)
|
|
||||||
self.session.commit()
|
|
||||||
version1 = user.versions[0]
|
|
||||||
|
|
||||||
handler = mod.FromWuttaToVersions(self.config)
|
|
||||||
handler.begin_target_transaction()
|
|
||||||
handler.target_session.close()
|
|
||||||
handler.target_session = self.session
|
|
||||||
|
|
||||||
imp = self.make_importer(
|
|
||||||
model_class=vercls,
|
|
||||||
fields=["uuid", "username"],
|
|
||||||
keys=("uuid",),
|
|
||||||
target_session=self.session,
|
|
||||||
continuum_txn=handler.continuum_txn,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(self.session.query(vercls).count(), 1)
|
|
||||||
self.assertIsNone(version1.end_transaction_id)
|
|
||||||
version2 = imp.delete_target_object(version1)
|
|
||||||
self.assertEqual(self.session.query(vercls).count(), 2)
|
|
||||||
self.assertEqual(version1.end_transaction_id, imp.continuum_txn.id)
|
|
||||||
self.assertEqual(version2.transaction_id, imp.continuum_txn.id)
|
|
||||||
self.assertEqual(version2.operation_type, continuum.Operation.DELETE)
|
|
||||||
self.assertIsNone(version2.end_transaction_id)
|
|
||||||
|
|
@ -1,3 +1,38 @@
|
||||||
# -*- coding: utf-8; -*-
|
# -*- coding: utf-8; -*-
|
||||||
|
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from wuttjamaican.testing import DataTestCase
|
||||||
|
|
||||||
from wuttasync.importing import wutta as mod
|
from wuttasync.importing import wutta as mod
|
||||||
|
|
||||||
|
|
||||||
|
class TestToWuttaHandler(DataTestCase):
|
||||||
|
|
||||||
|
def make_handler(self, **kwargs):
|
||||||
|
return mod.ToWuttaHandler(self.config, **kwargs)
|
||||||
|
|
||||||
|
def test_get_target_title(self):
|
||||||
|
handler = self.make_handler()
|
||||||
|
|
||||||
|
# uses app title by default
|
||||||
|
self.config.setdefault("wutta.app_title", "What About This")
|
||||||
|
self.assertEqual(handler.get_target_title(), "What About This")
|
||||||
|
|
||||||
|
# or generic default if present
|
||||||
|
handler.generic_target_title = "WHATABOUTTHIS"
|
||||||
|
self.assertEqual(handler.get_target_title(), "WHATABOUTTHIS")
|
||||||
|
|
||||||
|
# but prefer specific title if present
|
||||||
|
handler.target_title = "what_about_this"
|
||||||
|
self.assertEqual(handler.get_target_title(), "what_about_this")
|
||||||
|
|
||||||
|
def test_make_target_session(self):
|
||||||
|
handler = self.make_handler()
|
||||||
|
|
||||||
|
# makes "new" (mocked in our case) app session
|
||||||
|
with patch.object(self.app, "make_session") as make_session:
|
||||||
|
make_session.return_value = self.session
|
||||||
|
session = handler.make_target_session()
|
||||||
|
make_session.assert_called_once_with()
|
||||||
|
self.assertIs(session, self.session)
|
||||||
|
|
|
||||||
|
|
@ -1,126 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
|
|
||||||
from wuttjamaican.testing import ConfigTestCase
|
|
||||||
|
|
||||||
from wuttasync import app as mod
|
|
||||||
from wuttasync.importing import ImportHandler
|
|
||||||
from wuttasync.importing.csv import FromCsvToWutta
|
|
||||||
|
|
||||||
|
|
||||||
class FromFooToBar(ImportHandler):
|
|
||||||
source_key = "foo"
|
|
||||||
target_key = "bar"
|
|
||||||
|
|
||||||
|
|
||||||
class FromCsvToPoser(FromCsvToWutta):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class TestWuttaSyncAppProvider(ConfigTestCase):
|
|
||||||
|
|
||||||
def test_get_all_import_handlers(self):
|
|
||||||
|
|
||||||
# by default our custom handler is not found
|
|
||||||
handlers = self.app.get_all_import_handlers()
|
|
||||||
self.assertIn(FromCsvToWutta, handlers)
|
|
||||||
self.assertNotIn(FromFooToBar, handlers)
|
|
||||||
|
|
||||||
# make sure if we configure a custom handler, it is found
|
|
||||||
self.config.setdefault(
|
|
||||||
"wuttasync.importing.import.to_wutta.from_csv.handler",
|
|
||||||
"tests.test_app:FromFooToBar",
|
|
||||||
)
|
|
||||||
handlers = self.app.get_all_import_handlers()
|
|
||||||
self.assertIn(FromCsvToWutta, handlers)
|
|
||||||
self.assertIn(FromFooToBar, handlers)
|
|
||||||
|
|
||||||
def test_get_designated_import_handler_spec(self):
|
|
||||||
|
|
||||||
# fetch of unknown key returns none
|
|
||||||
spec = self.app.get_designated_import_handler_spec("test01")
|
|
||||||
self.assertIsNone(spec)
|
|
||||||
|
|
||||||
# unless we require it, in which case, error
|
|
||||||
self.assertRaises(
|
|
||||||
ValueError,
|
|
||||||
self.app.get_designated_import_handler_spec,
|
|
||||||
"test01",
|
|
||||||
require=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
# we configure one for whatever key we like
|
|
||||||
self.config.setdefault(
|
|
||||||
"wuttasync.importing.test02.handler", "tests.test_app:FromBarToFoo"
|
|
||||||
)
|
|
||||||
spec = self.app.get_designated_import_handler_spec("test02")
|
|
||||||
self.assertEqual(spec, "tests.test_app:FromBarToFoo")
|
|
||||||
|
|
||||||
# we can also define a "default" designated handler
|
|
||||||
self.config.setdefault(
|
|
||||||
"wuttasync.importing.test03.default_handler",
|
|
||||||
"tests.test_app:FromBarToFoo",
|
|
||||||
)
|
|
||||||
spec = self.app.get_designated_import_handler_spec("test03")
|
|
||||||
self.assertEqual(spec, "tests.test_app:FromBarToFoo")
|
|
||||||
|
|
||||||
def test_get_designated_import_handlers(self):
|
|
||||||
|
|
||||||
# some designated handlers exist, but not our custom handler
|
|
||||||
handlers = self.app.get_designated_import_handlers()
|
|
||||||
csv_handlers = [
|
|
||||||
h for h in handlers if h.get_key() == "import.to_wutta.from_csv"
|
|
||||||
]
|
|
||||||
self.assertEqual(len(csv_handlers), 1)
|
|
||||||
csv_handler = csv_handlers[0]
|
|
||||||
self.assertIsInstance(csv_handler, FromCsvToWutta)
|
|
||||||
self.assertFalse(isinstance(csv_handler, FromCsvToPoser))
|
|
||||||
self.assertFalse(
|
|
||||||
any([h.get_key() == "import.to_bar.from_foo" for h in handlers])
|
|
||||||
)
|
|
||||||
self.assertFalse(any([isinstance(h, FromFooToBar) for h in handlers]))
|
|
||||||
self.assertFalse(any([isinstance(h, FromCsvToPoser) for h in handlers]))
|
|
||||||
self.assertTrue(
|
|
||||||
any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
|
|
||||||
)
|
|
||||||
|
|
||||||
# but we can make custom designated
|
|
||||||
self.config.setdefault(
|
|
||||||
"wuttasync.importing.import.to_wutta.from_csv.handler",
|
|
||||||
"tests.test_app:FromCsvToPoser",
|
|
||||||
)
|
|
||||||
handlers = self.app.get_designated_import_handlers()
|
|
||||||
csv_handlers = [
|
|
||||||
h for h in handlers if h.get_key() == "import.to_wutta.from_csv"
|
|
||||||
]
|
|
||||||
self.assertEqual(len(csv_handlers), 1)
|
|
||||||
csv_handler = csv_handlers[0]
|
|
||||||
self.assertIsInstance(csv_handler, FromCsvToWutta)
|
|
||||||
self.assertIsInstance(csv_handler, FromCsvToPoser)
|
|
||||||
self.assertTrue(
|
|
||||||
any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_get_import_handler(self):
|
|
||||||
|
|
||||||
# make sure a basic fetch works
|
|
||||||
handler = self.app.get_import_handler("import.to_wutta.from_csv")
|
|
||||||
self.assertIsInstance(handler, FromCsvToWutta)
|
|
||||||
self.assertFalse(isinstance(handler, FromCsvToPoser))
|
|
||||||
|
|
||||||
# and make sure custom override works
|
|
||||||
self.config.setdefault(
|
|
||||||
"wuttasync.importing.import.to_wutta.from_csv.handler",
|
|
||||||
"tests.test_app:FromCsvToPoser",
|
|
||||||
)
|
|
||||||
handler = self.app.get_import_handler("import.to_wutta.from_csv")
|
|
||||||
self.assertIsInstance(handler, FromCsvToWutta)
|
|
||||||
self.assertIsInstance(handler, FromCsvToPoser)
|
|
||||||
|
|
||||||
# unknown importer cannot be found
|
|
||||||
handler = self.app.get_import_handler("bogus")
|
|
||||||
self.assertIsNone(handler)
|
|
||||||
|
|
||||||
# and if we require it, error will raise
|
|
||||||
self.assertRaises(
|
|
||||||
ValueError, self.app.get_import_handler, "bogus", require=True
|
|
||||||
)
|
|
||||||
|
|
@ -1,81 +0,0 @@
|
||||||
# -*- coding: utf-8; -*-
|
|
||||||
|
|
||||||
from wuttjamaican.testing import ConfigTestCase
|
|
||||||
|
|
||||||
from wuttasync import emails as mod
|
|
||||||
from wuttasync.importing import ImportHandler
|
|
||||||
from wuttasync.testing import ImportExportWarningTestCase
|
|
||||||
|
|
||||||
|
|
||||||
class FromFooToWutta(ImportHandler):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class TestImportExportWarning(ConfigTestCase):
|
|
||||||
|
|
||||||
def make_setting(self, factory=None):
|
|
||||||
if not factory:
|
|
||||||
factory = mod.ImportExportWarning
|
|
||||||
setting = factory(self.config)
|
|
||||||
return setting
|
|
||||||
|
|
||||||
def test_get_description(self):
|
|
||||||
self.config.setdefault("wutta.app_title", "Wutta Poser")
|
|
||||||
setting = self.make_setting()
|
|
||||||
setting.import_handler_key = "import.to_wutta.from_csv"
|
|
||||||
self.assertEqual(
|
|
||||||
setting.get_description(),
|
|
||||||
"Diff warning email for importing CSV → Wutta Poser",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_get_default_subject(self):
|
|
||||||
self.config.setdefault("wutta.app_title", "Wutta Poser")
|
|
||||||
setting = self.make_setting()
|
|
||||||
setting.import_handler_key = "import.to_wutta.from_csv"
|
|
||||||
self.assertEqual(setting.get_default_subject(), "Changes for CSV → Wutta Poser")
|
|
||||||
|
|
||||||
def test_get_import_handler(self):
|
|
||||||
|
|
||||||
# nb. typical name pattern
|
|
||||||
class import_to_wutta_from_foo_warning(mod.ImportExportWarning):
|
|
||||||
pass
|
|
||||||
|
|
||||||
# nb. name does not match spec pattern
|
|
||||||
class import_to_wutta_from_bar_blah(mod.ImportExportWarning):
|
|
||||||
pass
|
|
||||||
|
|
||||||
# register our import handler
|
|
||||||
self.config.setdefault(
|
|
||||||
"wuttasync.importing.import.to_wutta.from_foo.handler",
|
|
||||||
"tests.test_emails:FromFooToWutta",
|
|
||||||
)
|
|
||||||
|
|
||||||
# error if spec/key not discoverable
|
|
||||||
setting = self.make_setting(import_to_wutta_from_bar_blah)
|
|
||||||
self.assertRaises(ValueError, setting.get_import_handler)
|
|
||||||
|
|
||||||
# can lookup by name (auto-spec)
|
|
||||||
setting = self.make_setting(import_to_wutta_from_foo_warning)
|
|
||||||
handler = setting.get_import_handler()
|
|
||||||
self.assertIsInstance(handler, FromFooToWutta)
|
|
||||||
|
|
||||||
# can lookup by explicit spec
|
|
||||||
setting = self.make_setting(import_to_wutta_from_bar_blah)
|
|
||||||
setting.import_handler_spec = "tests.test_emails:FromFooToWutta"
|
|
||||||
handler = setting.get_import_handler()
|
|
||||||
self.assertIsInstance(handler, FromFooToWutta)
|
|
||||||
|
|
||||||
# can lookup by explicit key
|
|
||||||
setting = self.make_setting(import_to_wutta_from_bar_blah)
|
|
||||||
setting.import_handler_key = "import.to_wutta.from_foo"
|
|
||||||
handler = setting.get_import_handler()
|
|
||||||
self.assertIsInstance(handler, FromFooToWutta)
|
|
||||||
|
|
||||||
|
|
||||||
class TestEmailSettings(ImportExportWarningTestCase):
|
|
||||||
|
|
||||||
def test_import_to_versions_from_wutta_warning(self):
|
|
||||||
self.do_test_preview("import_to_versions_from_wutta_warning")
|
|
||||||
|
|
||||||
def test_import_to_wutta_from_csv_warning(self):
|
|
||||||
self.do_test_preview("import_to_wutta_from_csv_warning")
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue