Compare commits
12 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dec145ada5 | |||
| 30f119ffc7 | |||
| b1fdf488ad | |||
| 61deaad251 | |||
| c873cc462e | |||
| ead51bcd5a | |||
| 2ca7842e4f | |||
| 4cb3832213 | |||
| e397890098 | |||
| 6ee008e169 | |||
| c6d1822f3b | |||
| e037aece6a |
35 changed files with 1462 additions and 75 deletions
22
CHANGELOG.md
22
CHANGELOG.md
|
|
@ -5,6 +5,28 @@ All notable changes to WuttaSync will be documented in this file.
|
|||
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
||||
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## v0.5.0 (2026-01-03)
|
||||
|
||||
### Feat
|
||||
|
||||
- add support for `wutta export-csv` command
|
||||
|
||||
### Fix
|
||||
|
||||
- add `actioner` property for ImportHandler
|
||||
|
||||
## v0.4.0 (2025-12-31)
|
||||
|
||||
### Feat
|
||||
|
||||
- add support for `--comment` CLI param, to set versioning comment
|
||||
- add support for `--runas` CLI param, to set versioning authorship
|
||||
|
||||
### Fix
|
||||
|
||||
- make pylint happy
|
||||
- accept either `--recip` or `--recips` param for import commands
|
||||
|
||||
## v0.3.0 (2025-12-20)
|
||||
|
||||
### Feat
|
||||
|
|
|
|||
6
docs/api/wuttasync.cli.export_csv.rst
Normal file
6
docs/api/wuttasync.cli.export_csv.rst
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
``wuttasync.cli.export_csv``
|
||||
============================
|
||||
|
||||
.. automodule:: wuttasync.cli.export_csv
|
||||
:members:
|
||||
6
docs/api/wuttasync.exporting.base.rst
Normal file
6
docs/api/wuttasync.exporting.base.rst
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
``wuttasync.exporting.base``
|
||||
============================
|
||||
|
||||
.. automodule:: wuttasync.exporting.base
|
||||
:members:
|
||||
6
docs/api/wuttasync.exporting.csv.rst
Normal file
6
docs/api/wuttasync.exporting.csv.rst
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
``wuttasync.exporting.csv``
|
||||
===========================
|
||||
|
||||
.. automodule:: wuttasync.exporting.csv
|
||||
:members:
|
||||
6
docs/api/wuttasync.exporting.handlers.rst
Normal file
6
docs/api/wuttasync.exporting.handlers.rst
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
``wuttasync.exporting.handlers``
|
||||
================================
|
||||
|
||||
.. automodule:: wuttasync.exporting.handlers
|
||||
:members:
|
||||
6
docs/api/wuttasync.exporting.rst
Normal file
6
docs/api/wuttasync.exporting.rst
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
``wuttasync.exporting``
|
||||
=======================
|
||||
|
||||
.. automodule:: wuttasync.exporting
|
||||
:members:
|
||||
|
|
@ -73,9 +73,14 @@ cf. :doc:`rattail-manual:data/sync/index`.
|
|||
api/wuttasync.app
|
||||
api/wuttasync.cli
|
||||
api/wuttasync.cli.base
|
||||
api/wuttasync.cli.export_csv
|
||||
api/wuttasync.cli.import_csv
|
||||
api/wuttasync.cli.import_versions
|
||||
api/wuttasync.emails
|
||||
api/wuttasync.exporting
|
||||
api/wuttasync.exporting.base
|
||||
api/wuttasync.exporting.csv
|
||||
api/wuttasync.exporting.handlers
|
||||
api/wuttasync.importing
|
||||
api/wuttasync.importing.base
|
||||
api/wuttasync.importing.csv
|
||||
|
|
|
|||
|
|
@ -9,6 +9,24 @@ WuttaSync.
|
|||
It is fairly simple to add more; see :doc:`custom`.
|
||||
|
||||
|
||||
.. _wutta-export-csv:
|
||||
|
||||
``wutta export-csv``
|
||||
--------------------
|
||||
|
||||
Export data from the Wutta :term:`app database` to CSV file(s).
|
||||
|
||||
This *should* be able to automatically export any table mapped in the
|
||||
:term:`app model`. The only caveat is that it is "dumb" and does not
|
||||
have any special field handling. This means the column headers in the
|
||||
CSV will be the same as in the source table, and some data types may
|
||||
not behave as expected etc.
|
||||
|
||||
Defined in: :mod:`wuttasync.cli.export_csv`
|
||||
|
||||
.. program-output:: wutta export-csv --help
|
||||
|
||||
|
||||
.. _wutta-import-csv:
|
||||
|
||||
``wutta import-csv``
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ Here is the code and we'll explain below::
|
|||
config = ctx.parent.wutta_config
|
||||
handler = ImportCommandHandler(
|
||||
config, import_handler='poser.importing.foo:FromFooToPoser')
|
||||
handler.run(ctx.params)
|
||||
handler.run(ctx)
|
||||
|
||||
Hopefully it's straightforward but to be clear:
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "WuttaSync"
|
||||
version = "0.3.0"
|
||||
version = "0.5.0"
|
||||
description = "Wutta Framework for data import/export and real-time sync"
|
||||
readme = "README.md"
|
||||
authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]
|
||||
|
|
@ -28,8 +28,9 @@ requires-python = ">= 3.8"
|
|||
dependencies = [
|
||||
"humanize",
|
||||
"makefun",
|
||||
"rich",
|
||||
"SQLAlchemy-Utils",
|
||||
"WuttJamaican[db]>=0.27.0",
|
||||
"WuttJamaican[db]>=0.28.1",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -42,6 +43,7 @@ tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"]
|
|||
wuttasync = "wuttasync.app:WuttaSyncAppProvider"
|
||||
|
||||
[project.entry-points."wuttasync.importing"]
|
||||
"export.to_csv.from_wutta" = "wuttasync.exporting.csv:FromWuttaToCsv"
|
||||
"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions"
|
||||
"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta"
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
|
|
@ -26,12 +26,19 @@ WuttaSync - ``wutta`` subcommands
|
|||
This namespace exposes the following:
|
||||
|
||||
* :func:`~wuttasync.cli.base.import_command()`
|
||||
* :func:`~wuttasync.cli.base.file_export_command()`
|
||||
* :func:`~wuttasync.cli.base.file_import_command()`
|
||||
* :class:`~wuttasync.cli.base.ImportCommandHandler`
|
||||
"""
|
||||
|
||||
from .base import import_command, file_import_command, ImportCommandHandler
|
||||
from .base import (
|
||||
import_command,
|
||||
file_export_command,
|
||||
file_import_command,
|
||||
ImportCommandHandler,
|
||||
)
|
||||
|
||||
# nb. must bring in all modules for discovery to work
|
||||
from . import export_csv
|
||||
from . import import_csv
|
||||
from . import import_versions
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
|
|
@ -32,10 +32,12 @@ from typing import List, Optional
|
|||
from typing_extensions import Annotated
|
||||
|
||||
import makefun
|
||||
import rich
|
||||
import typer
|
||||
|
||||
from wuttjamaican.app import GenericHandler
|
||||
from wuttasync.importing import ImportHandler
|
||||
from wuttasync.importing import ImportHandler, FromFileHandler
|
||||
from wuttasync.exporting import ToFileHandler
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
@ -102,7 +104,7 @@ class ImportCommandHandler(GenericHandler):
|
|||
elif key:
|
||||
self.import_handler = self.app.get_import_handler(key, require=True)
|
||||
|
||||
def run(self, params, progress=None): # pylint: disable=unused-argument
|
||||
def run(self, ctx, progress=None): # pylint: disable=unused-argument
|
||||
"""
|
||||
Run the import/export job(s) based on command line params.
|
||||
|
||||
|
|
@ -113,21 +115,51 @@ class ImportCommandHandler(GenericHandler):
|
|||
Unless ``--list-models`` was specified on the command line in
|
||||
which case we do :meth:`list_models()` instead.
|
||||
|
||||
:param params: Dict of params from command line. This must
|
||||
include a ``'models'`` key, the rest are optional.
|
||||
:param ctx: :class:`typer.Context` instance.
|
||||
|
||||
:param progress: Optional progress indicator factory.
|
||||
"""
|
||||
|
||||
# maybe just list models and bail
|
||||
if params.get("list_models"):
|
||||
self.list_models(params)
|
||||
if ctx.params.get("list_models"):
|
||||
self.list_models(ctx.params)
|
||||
return
|
||||
|
||||
# otherwise process some data
|
||||
# otherwise we'll (hopefully) process some data
|
||||
log.debug("using handler: %s", self.import_handler.get_spec())
|
||||
kw = dict(params)
|
||||
models = kw.pop("models")
|
||||
|
||||
# but first, some extra checks for certain file-based
|
||||
# handlers. this must be done here, because these CLI params
|
||||
# are not technically required (otherwise typer would handle
|
||||
# this instead of us here). and that is because we want to
|
||||
# allow user to specify --list without needing to also specify
|
||||
# --input or --output
|
||||
if isinstance(self.import_handler, FromFileHandler):
|
||||
if not ctx.params.get("input_file_path"):
|
||||
rich.print(
|
||||
"\n[bold yellow]must specify --input folder/file path[/bold yellow]\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
elif isinstance(self.import_handler, ToFileHandler):
|
||||
if not ctx.params.get("output_file_path"):
|
||||
rich.print(
|
||||
"\n[bold yellow]must specify --output folder/file path[/bold yellow]\n",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# all params from caller will be passed along
|
||||
kw = dict(ctx.params)
|
||||
|
||||
# runas user and comment also, but they come from root command
|
||||
if username := ctx.parent.params.get("runas_username"):
|
||||
kw["runas_username"] = username
|
||||
if comment := ctx.parent.params.get("comment"):
|
||||
kw["transaction_comment"] = comment
|
||||
|
||||
# sort out which models to process
|
||||
models = kw.pop("models", None)
|
||||
if not models:
|
||||
models = list(self.import_handler.importers)
|
||||
log.debug(
|
||||
|
|
@ -136,6 +168,8 @@ class ImportCommandHandler(GenericHandler):
|
|||
self.import_handler.get_title(),
|
||||
", ".join(models),
|
||||
)
|
||||
|
||||
# process data
|
||||
log.debug("params are: %s", kw)
|
||||
self.import_handler.process_data(*models, **kw)
|
||||
|
||||
|
|
@ -159,7 +193,7 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
|
|||
models: Annotated[
|
||||
Optional[List[str]],
|
||||
typer.Argument(
|
||||
help="Model(s) to process. Can specify one or more, "
|
||||
help="Target model(s) to process. Specify one or more, "
|
||||
"or omit to process default models."
|
||||
),
|
||||
] = None,
|
||||
|
|
@ -248,7 +282,9 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
|
|||
warnings_recipients: Annotated[
|
||||
str,
|
||||
typer.Option(
|
||||
"--recip", help="Override the recipient(s) for diff warning email."
|
||||
"--recip",
|
||||
"--recips",
|
||||
help="Override the recipient(s) for diff warning email.",
|
||||
),
|
||||
] = None,
|
||||
warnings_max_diffs: Annotated[
|
||||
|
|
@ -311,17 +347,65 @@ def import_command(fn):
|
|||
return makefun.create_function(final_sig, fn)
|
||||
|
||||
|
||||
def file_import_command_template( # pylint: disable=unused-argument
|
||||
input_file_path: Annotated[
|
||||
def file_export_command_template( # pylint: disable=unused-argument
|
||||
# nb. technically this is required, but not if doing --list
|
||||
# (so we cannot mark it required here, for that reason)
|
||||
output_file_path: Annotated[
|
||||
Path,
|
||||
typer.Option(
|
||||
"--input-path",
|
||||
"--output",
|
||||
"-o",
|
||||
exists=True,
|
||||
file_okay=True,
|
||||
dir_okay=True,
|
||||
help="Path to input file(s). Can be a folder "
|
||||
"if app logic can guess the filename(s); "
|
||||
"otherwise must be complete file path.",
|
||||
help="Path to output folder. Or full path to output file "
|
||||
"if only running one target model.",
|
||||
),
|
||||
] = None,
|
||||
):
|
||||
"""
|
||||
Stub function to provide signature for exporter commands which
|
||||
produce data file(s) as output. Used with
|
||||
:func:`file_export_command`.
|
||||
"""
|
||||
|
||||
|
||||
def file_export_command(fn):
|
||||
"""
|
||||
Decorator for file export commands. Adds common params based on
|
||||
:func:`file_export_command_template`.
|
||||
"""
|
||||
original_sig = inspect.signature(fn)
|
||||
plain_import_sig = inspect.signature(import_command_template)
|
||||
file_export_sig = inspect.signature(file_export_command_template)
|
||||
desired_params = list(plain_import_sig.parameters.values()) + list(
|
||||
file_export_sig.parameters.values()
|
||||
)
|
||||
|
||||
params = list(original_sig.parameters.values())
|
||||
for i, param in enumerate(desired_params):
|
||||
params.insert(i + 1, param)
|
||||
|
||||
# remove the **kwargs param
|
||||
params.pop(-1)
|
||||
|
||||
final_sig = original_sig.replace(parameters=params)
|
||||
return makefun.create_function(final_sig, fn)
|
||||
|
||||
|
||||
def file_import_command_template( # pylint: disable=unused-argument
|
||||
# nb. technically this is required, but not if doing --list
|
||||
# (so we cannot mark it required here, for that reason)
|
||||
input_file_path: Annotated[
|
||||
Path,
|
||||
typer.Option(
|
||||
"--input",
|
||||
"-i",
|
||||
exists=True,
|
||||
file_okay=True,
|
||||
dir_okay=True,
|
||||
help="Path to input folder. Or full path to input file "
|
||||
"if only running one target model.",
|
||||
),
|
||||
] = None,
|
||||
):
|
||||
|
|
|
|||
42
src/wuttasync/cli/export_csv.py
Normal file
42
src/wuttasync/cli/export_csv.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
See also: :ref:`wutta-export-csv`
|
||||
"""
|
||||
|
||||
import typer
|
||||
|
||||
from wuttjamaican.cli import wutta_typer
|
||||
|
||||
from .base import file_export_command, ImportCommandHandler
|
||||
|
||||
|
||||
@wutta_typer.command()
|
||||
@file_export_command
|
||||
def export_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
|
||||
"""
|
||||
Export data from Wutta DB to CSV file(s)
|
||||
"""
|
||||
config = ctx.parent.wutta_config
|
||||
handler = ImportCommandHandler(config, key="export.to_csv.from_wutta")
|
||||
handler.run(ctx)
|
||||
|
|
@ -39,4 +39,4 @@ def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
|
|||
"""
|
||||
config = ctx.parent.wutta_config
|
||||
handler = ImportCommandHandler(config, key="import.to_wutta.from_csv")
|
||||
handler.run(ctx.params)
|
||||
handler.run(ctx)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import sys
|
|||
|
||||
import rich
|
||||
import typer
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from wuttjamaican.cli import wutta_typer
|
||||
|
||||
|
|
@ -37,14 +36,7 @@ from .base import import_command, ImportCommandHandler
|
|||
|
||||
@wutta_typer.command()
|
||||
@import_command
|
||||
def import_versions( # pylint: disable=unused-argument
|
||||
ctx: typer.Context,
|
||||
comment: Annotated[
|
||||
str,
|
||||
typer.Option("--comment", "-m", help="Comment to set on the transaction."),
|
||||
] = "import catch-up versions",
|
||||
**kwargs,
|
||||
):
|
||||
def import_versions(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
|
||||
"""
|
||||
Import latest data to version tables, for Wutta DB
|
||||
"""
|
||||
|
|
@ -70,4 +62,4 @@ def import_versions( # pylint: disable=unused-argument
|
|||
sys.exit(1)
|
||||
|
||||
handler = ImportCommandHandler(config, key="import.to_versions.from_wutta")
|
||||
handler.run(ctx.params)
|
||||
handler.run(ctx)
|
||||
|
|
|
|||
43
src/wuttasync/exporting/__init__.py
Normal file
43
src/wuttasync/exporting/__init__.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Data Import / Export Framework
|
||||
|
||||
This namespace exposes the following:
|
||||
|
||||
* :enum:`~wuttasync.importing.handlers.Orientation`
|
||||
|
||||
And some :term:`export handler <import handler>` base classes:
|
||||
|
||||
* :class:`~wuttasync.exporting.handlers.ExportHandler`
|
||||
* :class:`~wuttasync.exporting.handlers.ToFileHandler`
|
||||
|
||||
And some :term:`exporter <importer>` base classes:
|
||||
|
||||
* :class:`~wuttasync.exporting.base.ToFile`
|
||||
|
||||
See also the :mod:`wuttasync.importing` module.
|
||||
"""
|
||||
|
||||
from .handlers import Orientation, ExportHandler, ToFileHandler
|
||||
from .base import ToFile
|
||||
166
src/wuttasync/exporting/base.py
Normal file
166
src/wuttasync/exporting/base.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Data exporter base classes
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from wuttasync.importing import Importer
|
||||
|
||||
|
||||
class ToFile(Importer):
|
||||
"""
|
||||
Base class for importer/exporter using output file as data target.
|
||||
|
||||
Depending on the subclass, it may be able to "guess" (at least
|
||||
partially) the path to the output file. If not, and/or to avoid
|
||||
ambiguity, the caller must specify the file path.
|
||||
|
||||
In most cases caller may specify any of these via kwarg to the
|
||||
class constructor, or e.g.
|
||||
:meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`:
|
||||
|
||||
* :attr:`output_file_path`
|
||||
* :attr:`output_file_name`
|
||||
|
||||
The subclass itself can also specify via override of these
|
||||
methods:
|
||||
|
||||
* :meth:`get_output_file_path()`
|
||||
* :meth:`get_output_file_name()`
|
||||
|
||||
And of course subclass must override these too:
|
||||
|
||||
* :meth:`open_output_file()`
|
||||
* :meth:`close_output_file()`
|
||||
* (and see also :attr:`output_file`)
|
||||
"""
|
||||
|
||||
output_file_path = None
|
||||
"""
|
||||
Path to output folder, or file.
|
||||
|
||||
The ideal usage is to set this to the output *folder* path. That
|
||||
allows the handler to run several importers in one go. The same
|
||||
output folder path is given to each importer; they then each
|
||||
determine their own output filename within that.
|
||||
|
||||
But you can also set this to the full output folder + file path,
|
||||
e.g. if you're just running one importer. This would override
|
||||
the importer's own logic for determining output filename.
|
||||
|
||||
See also :meth:`get_output_file_path()` and
|
||||
:meth:`get_output_file_name()`.
|
||||
"""
|
||||
|
||||
output_file_name = None
|
||||
"""
|
||||
Optional static output file name (sans folder path).
|
||||
|
||||
If set, this will be used as output filename instead of the
|
||||
importer determining one on its own.
|
||||
|
||||
See also :meth:`get_output_file_name()`.
|
||||
"""
|
||||
|
||||
output_file = None
|
||||
"""
|
||||
Handle to the open output file, if applicable. May be set by
|
||||
:meth:`open_output_file()` for later reference within
|
||||
:meth:`close_output_file()`.
|
||||
"""
|
||||
|
||||
def setup(self):
|
||||
"""
|
||||
Open the output file. See also :meth:`open_output_file()`.
|
||||
"""
|
||||
if not self.dry_run:
|
||||
self.open_output_file()
|
||||
|
||||
def teardown(self):
|
||||
"""
|
||||
Close the output file. See also :meth:`close_output_file()`.
|
||||
"""
|
||||
if not self.dry_run:
|
||||
self.close_output_file()
|
||||
|
||||
def get_output_file_path(self):
|
||||
"""
|
||||
This must return the full path to output file.
|
||||
|
||||
Default logic inspects :attr:`output_file_path`; if that
|
||||
points to a folder then it is combined with
|
||||
:meth:`get_output_file_name()`. Otherwise it's returned
|
||||
as-is.
|
||||
|
||||
:returns: Path to output file, as string
|
||||
"""
|
||||
path = self.output_file_path
|
||||
if not path:
|
||||
raise ValueError("must set output_file_path")
|
||||
|
||||
if os.path.isdir(path):
|
||||
filename = self.get_output_file_name()
|
||||
return os.path.join(path, filename)
|
||||
|
||||
return path
|
||||
|
||||
def get_output_file_name(self):
|
||||
"""
|
||||
This must return the output filename, sans folder path.
|
||||
|
||||
Default logic will return :attr:`output_file_name` if set,
|
||||
otherwise raise error.
|
||||
|
||||
:returns: Output filename, sans folder path
|
||||
"""
|
||||
if self.output_file_name:
|
||||
return self.output_file_name
|
||||
|
||||
raise NotImplementedError("can't guess output filename")
|
||||
|
||||
def open_output_file(self):
|
||||
"""
|
||||
Open the output file for writing target data.
|
||||
|
||||
Subclass must override to specify how this happens; default
|
||||
logic is not implemented. Remember to set :attr:`output_file`
|
||||
if applicable for reference when closing.
|
||||
|
||||
See also :attr:`get_output_file_path()` and
|
||||
:meth:`close_output_file()`.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def close_output_file(self):
|
||||
"""
|
||||
Close the output file for target data.
|
||||
|
||||
Subclass must override to specify how this happens; default
|
||||
logic blindly calls the ``close()`` method on whatever
|
||||
:attr:`output_file` happens to point to.
|
||||
|
||||
See also :attr:`open_output_file()`.
|
||||
"""
|
||||
self.output_file.close()
|
||||
321
src/wuttasync/exporting/csv.py
Normal file
321
src/wuttasync/exporting/csv.py
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Exporting to CSV
|
||||
"""
|
||||
|
||||
import csv
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy_utils.functions import get_primary_keys, get_columns
|
||||
|
||||
from wuttjamaican.db.util import make_topo_sortkey
|
||||
|
||||
from wuttasync.importing import FromWuttaHandler, FromWutta
|
||||
from wuttasync.exporting import ToFileHandler, ToFile
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ToCsv(ToFile): # pylint: disable=abstract-method
|
||||
"""
|
||||
Base class for exporter using CSV file as data target.
|
||||
|
||||
This inherits from :class:`~wuttasync.exporting.base.ToFile`.
|
||||
"""
|
||||
|
||||
output_writer = None
|
||||
"""
|
||||
While the output file is open, this will reference a
|
||||
:class:`python:csv.DictWriter` instance.
|
||||
"""
|
||||
|
||||
csv_encoding = "utf_8"
|
||||
"""
|
||||
Encoding used for the CSV output file.
|
||||
|
||||
You can specify an override if needed when calling
|
||||
:meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`.
|
||||
"""
|
||||
|
||||
def get_output_file_name(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
if self.output_file_name:
|
||||
return self.output_file_name
|
||||
|
||||
model_title = self.get_model_title()
|
||||
return f"{model_title}.csv"
|
||||
|
||||
def open_output_file(self):
|
||||
"""
|
||||
Opens the output CSV file for writing.
|
||||
|
||||
This calls
|
||||
:meth:`~wuttasync.exporting.base.ToFile.get_output_file_path()`
|
||||
and opens that file. It sets
|
||||
:attr:`~wuttasync.exporting.base.ToFile.output_file` and also
|
||||
:attr:`output_writer`. And it calls
|
||||
:meth:`write_output_header()` to write the field header row.
|
||||
"""
|
||||
path = self.get_output_file_path()
|
||||
log.debug("opening output file: %s", path)
|
||||
|
||||
self.output_file = open( # pylint: disable=consider-using-with
|
||||
path, "wt", encoding=self.csv_encoding
|
||||
)
|
||||
|
||||
self.output_writer = csv.DictWriter(
|
||||
self.output_file,
|
||||
self.fields,
|
||||
# quoting=csv.QUOTE_NONNUMERIC
|
||||
)
|
||||
|
||||
self.write_output_header()
|
||||
|
||||
def write_output_header(self):
|
||||
"""
|
||||
Write the field header row to the CSV file.
|
||||
|
||||
Default logic calls
|
||||
:meth:`~python:csv.DictWriter.writeheader()` on the
|
||||
:attr:`output_writer` instance.
|
||||
"""
|
||||
self.output_writer.writeheader()
|
||||
|
||||
def close_output_file(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
self.output_writer = None
|
||||
self.output_file.close()
|
||||
self.output_file = None
|
||||
|
||||
def update_target_object(self, obj, source_data, target_data=None):
|
||||
"""
|
||||
In a CSV export the assumption is we always start with an
|
||||
empty file, so "create" is the only logical action for each
|
||||
record - there are no updates or deletes per se.
|
||||
|
||||
But under the hood, this method is used for create as well, so
|
||||
we override it and actually write the record to CSV file.
|
||||
Unless :attr:`~wuttasync.importing.base.Importer.dry_run` is
|
||||
true, this calls :meth:`~python:csv.csvwriter.writerow()` on
|
||||
the :attr:`output_writer` instance.
|
||||
|
||||
See also parent method docs,
|
||||
:meth:`~wuttasync.importing.base.Importer.update_target_object()`
|
||||
"""
|
||||
data = self.coerce_csv(source_data)
|
||||
if not self.dry_run:
|
||||
self.output_writer.writerow(data)
|
||||
return data
|
||||
|
||||
def coerce_csv(self, data): # pylint: disable=missing-function-docstring
|
||||
coerced = {}
|
||||
for field in self.fields:
|
||||
value = data[field]
|
||||
|
||||
if value is None:
|
||||
value = ""
|
||||
|
||||
elif isinstance(value, (int, float)):
|
||||
pass
|
||||
|
||||
else:
|
||||
value = str(value)
|
||||
|
||||
coerced[field] = value
|
||||
return coerced
|
||||
|
||||
|
||||
class FromSqlalchemyToCsvMixin:
|
||||
"""
|
||||
Mixin class for SQLAlchemy ORM → CSV :term:`exporters <importer>`.
|
||||
|
||||
Such exporters are generated automatically by
|
||||
:class:`FromSqlalchemyToCsvHandlerMixin`, so you won't typically
|
||||
reference this mixin class directly.
|
||||
|
||||
This mixin effectively behaves like the
|
||||
:attr:`~wuttasync.importing.base.Importer.model_class` represents
|
||||
the source side instead of the target. It uses
|
||||
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class`
|
||||
instead, for automatic things like inspecting the fields list.
|
||||
"""
|
||||
|
||||
def get_model_title(self): # pylint: disable=missing-function-docstring
|
||||
if hasattr(self, "model_title"):
|
||||
return self.model_title
|
||||
return self.source_model_class.__name__
|
||||
|
||||
def get_simple_fields(self): # pylint: disable=missing-function-docstring
|
||||
if hasattr(self, "simple_fields"):
|
||||
return self.simple_fields
|
||||
try:
|
||||
fields = get_columns(self.source_model_class)
|
||||
except sa.exc.NoInspectionAvailable:
|
||||
return []
|
||||
return list(fields.keys())
|
||||
|
||||
def normalize_source_object(
|
||||
self, obj
|
||||
): # pylint: disable=missing-function-docstring
|
||||
fields = self.get_fields()
|
||||
fields = [f for f in self.get_simple_fields() if f in fields]
|
||||
data = {field: getattr(obj, field) for field in fields}
|
||||
return data
|
||||
|
||||
def make_object(self): # pylint: disable=missing-function-docstring
|
||||
return self.source_model_class()
|
||||
|
||||
|
||||
class FromSqlalchemyToCsvHandlerMixin:
|
||||
"""
|
||||
Mixin class for SQLAlchemy ORM → CSV :term:`export handlers
|
||||
<import handler>`.
|
||||
|
||||
This knows how to dynamically generate :term:`exporter <importer>`
|
||||
classes to represent the models in the source ORM. Such classes
|
||||
will inherit from :class:`FromSqlalchemyToCsvMixin`, in addition
|
||||
to whatever :attr:`FromImporterBase` and :attr:`ToImporterBase`
|
||||
reference.
|
||||
|
||||
That all happens within :meth:`define_importers()`.
|
||||
"""
|
||||
|
||||
target_key = "csv"
|
||||
generic_target_title = "CSV"
|
||||
|
||||
# nb. subclass must define this
|
||||
FromImporterBase = None
|
||||
"""
|
||||
For a handler to use this mixin, it must set this to a valid base
|
||||
class for the ORM source side. The :meth:`define_importers()`
|
||||
logic will use this when dynamically generating new exporter
|
||||
classes.
|
||||
"""
|
||||
|
||||
ToImporterBase = ToCsv
|
||||
"""
|
||||
This must be set to a valid base class for the CSV target side.
|
||||
Default is :class:`ToCsv` which should typically be fine; you can
|
||||
change if needed.
|
||||
"""
|
||||
|
||||
def get_source_model(self):
|
||||
"""
|
||||
This should return the :term:`app model` or a similar module
|
||||
containing data model classes for the source side.
|
||||
|
||||
The source model is used to dynamically generate a set of
|
||||
exporters (e.g. one per table in the source DB) which can use
|
||||
CSV file as data target. See also :meth:`define_importers()`.
|
||||
|
||||
Subclass must override this if needed; default behavior is not
|
||||
implemented.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def define_importers(self):
|
||||
"""
|
||||
This mixin overrides typical (manual) importer definition, and
|
||||
instead dynamically generates a set of exporters, e.g. one per
|
||||
table in the source DB.
|
||||
|
||||
It does this based on the source model, as returned by
|
||||
:meth:`get_source_model()`. It calls
|
||||
:meth:`make_importer_factory()` for each model class found.
|
||||
"""
|
||||
importers = {}
|
||||
model = self.get_source_model()
|
||||
|
||||
# mostly try to make an importer for every data model
|
||||
for name in dir(model):
|
||||
cls = getattr(model, name)
|
||||
if (
|
||||
isinstance(cls, type)
|
||||
and issubclass(cls, model.Base)
|
||||
and cls is not model.Base
|
||||
):
|
||||
importers[name] = self.make_importer_factory(cls, name)
|
||||
|
||||
# sort importers according to schema topography
|
||||
topo_sortkey = make_topo_sortkey(model)
|
||||
importers = OrderedDict(
|
||||
[(name, importers[name]) for name in sorted(importers, key=topo_sortkey)]
|
||||
)
|
||||
|
||||
return importers
|
||||
|
||||
def make_importer_factory(self, model_class, name):
|
||||
"""
|
||||
Generate a new :term:`exporter <importer>` class, targeting
|
||||
the given :term:`data model` class.
|
||||
|
||||
The newly-created class will inherit from:
|
||||
|
||||
* :class:`FromSqlalchemyToCsvMixin`
|
||||
* :attr:`FromImporterBase`
|
||||
* :attr:`ToImporterBase`
|
||||
|
||||
:param model_class: A data model class.
|
||||
|
||||
:param name: The "model name" for the importer/exporter. New
|
||||
class name will be based on this, so e.g. ``Widget`` model
|
||||
name becomes ``WidgetImporter`` class name.
|
||||
|
||||
:returns: The new class, meant to process import/export
|
||||
targeting the given data model.
|
||||
"""
|
||||
return type(
|
||||
f"{name}Importer",
|
||||
(FromSqlalchemyToCsvMixin, self.FromImporterBase, self.ToImporterBase),
|
||||
{
|
||||
"source_model_class": model_class,
|
||||
"default_keys": list(get_primary_keys(model_class)),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class ToCsvHandler(ToFileHandler):
|
||||
"""
|
||||
Base class for export handlers using CSV file(s) as data target.
|
||||
"""
|
||||
|
||||
|
||||
class FromWuttaToCsv(
|
||||
FromSqlalchemyToCsvHandlerMixin, FromWuttaHandler, ToCsvHandler
|
||||
): # pylint: disable=too-many-ancestors
|
||||
"""
|
||||
Handler for Wutta (:term:`app database`) → CSV export.
|
||||
|
||||
This uses :class:`FromSqlalchemyToCsvHandlerMixin` for most of the
|
||||
heavy lifting.
|
||||
"""
|
||||
|
||||
FromImporterBase = FromWutta
|
||||
|
||||
def get_source_model(self): # pylint: disable=empty-docstring
|
||||
""" """
|
||||
return self.app.model
|
||||
50
src/wuttasync/exporting/handlers.py
Normal file
50
src/wuttasync/exporting/handlers.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
# Wutta Framework is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by the Free
|
||||
# Software Foundation, either version 3 of the License, or (at your option) any
|
||||
# later version.
|
||||
#
|
||||
# Wutta Framework is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
# more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
################################################################################
|
||||
"""
|
||||
Export Handlers
|
||||
"""
|
||||
|
||||
from wuttasync.importing import ImportHandler, Orientation
|
||||
|
||||
|
||||
class ExportHandler(ImportHandler):
|
||||
"""
|
||||
Generic base class for :term:`export handlers <import handler>`.
|
||||
|
||||
This is really just
|
||||
:class:`~wuttasync.importing.handlers.ImportHandler` with the
|
||||
orientation flipped.
|
||||
"""
|
||||
|
||||
orientation = Orientation.EXPORT
|
||||
"" # nb. suppress docs
|
||||
|
||||
|
||||
class ToFileHandler(ExportHandler):
|
||||
"""
|
||||
Base class for export handlers which use output file(s) as the
|
||||
data target.
|
||||
|
||||
Importers (exporters) used by this handler are generally assumed
|
||||
to subclass :class:`~wuttasync.exporting.base.ToFile`.
|
||||
"""
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
|
|
@ -31,6 +31,8 @@ And some :term:`import handler` base classes:
|
|||
|
||||
* :class:`~wuttasync.importing.handlers.ImportHandler`
|
||||
* :class:`~wuttasync.importing.handlers.FromFileHandler`
|
||||
* :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`
|
||||
* :class:`~wuttasync.importing.handlers.FromWuttaHandler`
|
||||
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
|
||||
* :class:`~wuttasync.importing.handlers.ToWuttaHandler`
|
||||
|
||||
|
|
@ -38,16 +40,22 @@ And some :term:`importer` base classes:
|
|||
|
||||
* :class:`~wuttasync.importing.base.Importer`
|
||||
* :class:`~wuttasync.importing.base.FromFile`
|
||||
* :class:`~wuttasync.importing.base.FromSqlalchemy`
|
||||
* :class:`~wuttasync.importing.base.FromWutta`
|
||||
* :class:`~wuttasync.importing.base.ToSqlalchemy`
|
||||
* :class:`~wuttasync.importing.model.ToWutta`
|
||||
|
||||
See also the :mod:`wuttasync.exporting` module.
|
||||
"""
|
||||
|
||||
from .handlers import (
|
||||
Orientation,
|
||||
ImportHandler,
|
||||
FromFileHandler,
|
||||
FromSqlalchemyHandler,
|
||||
FromWuttaHandler,
|
||||
ToSqlalchemyHandler,
|
||||
ToWuttaHandler,
|
||||
)
|
||||
from .base import Importer, FromFile, ToSqlalchemy
|
||||
from .base import Importer, FromFile, FromSqlalchemy, FromWutta, ToSqlalchemy
|
||||
from .model import ToWutta
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
################################################################################
|
||||
#
|
||||
# WuttaSync -- Wutta Framework for data import/export and real-time sync
|
||||
# Copyright © 2024-2025 Lance Edgar
|
||||
# Copyright © 2024-2026 Lance Edgar
|
||||
#
|
||||
# This file is part of Wutta Framework.
|
||||
#
|
||||
|
|
@ -1448,6 +1448,13 @@ class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method
|
|||
return self.normalize_target_object(obj)
|
||||
|
||||
|
||||
class FromWutta(FromSqlalchemy): # pylint: disable=abstract-method
|
||||
"""
|
||||
Base class for data importer/exporter which uses the Wutta ORM
|
||||
(:term:`app database`) as data source.
|
||||
"""
|
||||
|
||||
|
||||
class ToSqlalchemy(Importer):
|
||||
"""
|
||||
Base class for importer/exporter which uses SQLAlchemy ORM on the
|
||||
|
|
|
|||
|
|
@ -391,6 +391,10 @@ def make_coercer(attr): # pylint: disable=too-many-return-statements
|
|||
):
|
||||
return coerce_datetime
|
||||
|
||||
# Date
|
||||
if isinstance(attr.type, sa.Date):
|
||||
return coerce_date
|
||||
|
||||
# Float
|
||||
# nb. check this before decimal, since Numeric inherits from Float
|
||||
if isinstance(attr.type, sa.Float):
|
||||
|
|
@ -423,6 +427,13 @@ def coerce_boolean_nullable(value): # pylint: disable=missing-function-docstrin
|
|||
return coerce_boolean(value)
|
||||
|
||||
|
||||
def coerce_date(value): # pylint: disable=missing-function-docstring
|
||||
if value == "":
|
||||
return None
|
||||
|
||||
return datetime.datetime.strptime(value, "%Y-%m-%d").date()
|
||||
|
||||
|
||||
def coerce_datetime(value): # pylint: disable=missing-function-docstring
|
||||
if value == "":
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
"""
|
||||
Data Import / Export Handlers
|
||||
"""
|
||||
# pylint: disable=too-many-lines
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -48,7 +49,9 @@ class Orientation(Enum):
|
|||
EXPORT = "export"
|
||||
|
||||
|
||||
class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
||||
class ImportHandler( # pylint: disable=too-many-public-methods,too-many-instance-attributes
|
||||
GenericHandler
|
||||
):
|
||||
"""
|
||||
Base class for all import/export handlers.
|
||||
|
||||
|
|
@ -166,6 +169,18 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
See also :attr:`warnings`.
|
||||
"""
|
||||
|
||||
runas_username = None
|
||||
"""
|
||||
Username responsible for running the import/export job. This is
|
||||
mostly used for Continuum versioning.
|
||||
"""
|
||||
|
||||
transaction_comment = None
|
||||
"""
|
||||
Optional comment to apply to the transaction, where applicable.
|
||||
This is mostly used for Continuum versioning.
|
||||
"""
|
||||
|
||||
importers = None
|
||||
"""
|
||||
This should be a dict of all importer/exporter classes available
|
||||
|
|
@ -195,6 +210,19 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
""" """
|
||||
return self.get_title()
|
||||
|
||||
@property
|
||||
def actioner(self):
|
||||
"""
|
||||
Convenience property which effectively returns the
|
||||
:attr:`orientation` as a noun - i.e. one of:
|
||||
|
||||
* ``'importer'``
|
||||
* ``'exporter'``
|
||||
|
||||
See also :attr:`actioning`.
|
||||
"""
|
||||
return f"{self.orientation.value}er"
|
||||
|
||||
@property
|
||||
def actioning(self):
|
||||
"""
|
||||
|
|
@ -203,6 +231,8 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
|
||||
* ``'importing'``
|
||||
* ``'exporting'``
|
||||
|
||||
See also :attr:`actioner`.
|
||||
"""
|
||||
return f"{self.orientation.value}ing"
|
||||
|
||||
|
|
@ -416,6 +446,12 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
|
|||
if "warnings_max_diffs" in kwargs:
|
||||
self.warnings_max_diffs = kwargs.pop("warnings_max_diffs")
|
||||
|
||||
if "runas_username" in kwargs:
|
||||
self.runas_username = kwargs.pop("runas_username")
|
||||
|
||||
if "transaction_comment" in kwargs:
|
||||
self.transaction_comment = kwargs.pop("transaction_comment")
|
||||
|
||||
return kwargs
|
||||
|
||||
def begin_transaction(self):
|
||||
|
|
@ -946,11 +982,41 @@ class ToWuttaHandler(ToSqlalchemyHandler):
|
|||
|
||||
def make_target_session(self):
|
||||
"""
|
||||
Call
|
||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
|
||||
and return the result.
|
||||
This creates a typical :term:`db session` for the app by
|
||||
calling
|
||||
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`.
|
||||
|
||||
It then may "customize" the session slightly. These
|
||||
customizations only are relevant if Wutta-Continuum versioning
|
||||
is enabled:
|
||||
|
||||
If :attr:`~ImportHandler.runas_username` is set, the
|
||||
responsible user (``continuum_user_id``) will be set for the
|
||||
new session as well.
|
||||
|
||||
Similarly, if :attr:`~ImportHandler.transaction_comment` is
|
||||
set, it (``continuum_comment``) will also be set for the new
|
||||
session.
|
||||
|
||||
:returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
|
||||
instance.
|
||||
"""
|
||||
return self.app.make_session()
|
||||
model = self.app.model
|
||||
session = self.app.make_session()
|
||||
|
||||
# set runas user in case continuum versioning is enabled
|
||||
if self.runas_username:
|
||||
if user := (
|
||||
session.query(model.User)
|
||||
.filter_by(username=self.runas_username)
|
||||
.first()
|
||||
):
|
||||
session.info["continuum_user_id"] = user.uuid
|
||||
else:
|
||||
log.warning("runas username not found: %s", self.runas_username)
|
||||
|
||||
# set comment in case continuum versioning is enabled
|
||||
if self.transaction_comment:
|
||||
session.info["continuum_comment"] = self.transaction_comment
|
||||
|
||||
return session
|
||||
|
|
|
|||
|
|
@ -92,13 +92,6 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
|
|||
See also :attr:`continuum_uow`.
|
||||
"""
|
||||
|
||||
continuum_comment = None
|
||||
|
||||
def consume_kwargs(self, kwargs):
|
||||
kwargs = super().consume_kwargs(kwargs)
|
||||
self.continuum_comment = kwargs.pop("comment", None)
|
||||
return kwargs
|
||||
|
||||
def begin_target_transaction(self):
|
||||
# pylint: disable=line-too-long
|
||||
"""
|
||||
|
|
@ -128,8 +121,8 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
|
|||
|
||||
self.continuum_txn = self.continuum_uow.create_transaction(self.target_session)
|
||||
|
||||
if self.continuum_comment:
|
||||
self.continuum_txn.meta = {"comment": self.continuum_comment}
|
||||
if self.transaction_comment:
|
||||
self.continuum_txn.meta = {"comment": self.transaction_comment}
|
||||
|
||||
def get_importer_kwargs(self, key, **kwargs):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
import inspect
|
||||
import sys
|
||||
from unittest import TestCase
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import patch, Mock
|
||||
|
||||
from wuttasync.cli import base as mod
|
||||
from wuttjamaican.testing import DataTestCase
|
||||
|
|
@ -44,12 +45,97 @@ class TestImportCommandHandler(DataTestCase):
|
|||
)
|
||||
|
||||
with patch.object(handler, "list_models") as list_models:
|
||||
handler.run({"list_models": True})
|
||||
list_models.assert_called_once_with({"list_models": True})
|
||||
ctx = Mock(params={"list_models": True})
|
||||
handler.run(ctx)
|
||||
list_models.assert_called_once_with(ctx.params)
|
||||
|
||||
class Object:
|
||||
def __init__(self, **kw):
|
||||
self.__dict__.update(kw)
|
||||
|
||||
with patch.object(handler, "import_handler") as import_handler:
|
||||
handler.run({"models": []})
|
||||
import_handler.process_data.assert_called_once_with()
|
||||
parent = Mock(
|
||||
params={
|
||||
"runas_username": "fred",
|
||||
"comment": "hello world",
|
||||
}
|
||||
)
|
||||
# TODO: why can't we just use Mock here? the parent attr is problematic
|
||||
ctx = Object(params={"models": []}, parent=parent)
|
||||
handler.run(ctx)
|
||||
import_handler.process_data.assert_called_once_with(
|
||||
runas_username="fred",
|
||||
transaction_comment="hello world",
|
||||
)
|
||||
|
||||
def test_run_missing_input(self):
|
||||
handler = self.make_handler(
|
||||
import_handler="wuttasync.importing.csv:FromCsvToWutta"
|
||||
)
|
||||
|
||||
class Object:
|
||||
def __init__(self, **kw):
|
||||
self.__dict__.update(kw)
|
||||
|
||||
# fails without input_file_path
|
||||
with patch.object(sys, "exit") as exit_:
|
||||
exit_.side_effect = RuntimeError
|
||||
ctx = Object(
|
||||
params={},
|
||||
parent=Object(params={}),
|
||||
)
|
||||
try:
|
||||
handler.run(ctx)
|
||||
except RuntimeError:
|
||||
pass
|
||||
exit_.assert_called_once_with(1)
|
||||
|
||||
# runs with input_file_path
|
||||
with patch.object(sys, "exit") as exit_:
|
||||
exit_.side_effect = RuntimeError
|
||||
ctx = Object(
|
||||
params={"input_file_path": self.tempdir},
|
||||
parent=Object(
|
||||
params={},
|
||||
),
|
||||
)
|
||||
self.assertRaises(FileNotFoundError, handler.run, ctx)
|
||||
exit_.assert_not_called()
|
||||
|
||||
def test_run_missing_output(self):
|
||||
handler = self.make_handler(
|
||||
import_handler="wuttasync.exporting.csv:FromWuttaToCsv"
|
||||
)
|
||||
|
||||
class Object:
|
||||
def __init__(self, **kw):
|
||||
self.__dict__.update(kw)
|
||||
|
||||
# fails without output_file_path
|
||||
with patch.object(sys, "exit") as exit_:
|
||||
exit_.side_effect = RuntimeError
|
||||
ctx = Object(
|
||||
params={},
|
||||
parent=Object(params={}),
|
||||
)
|
||||
try:
|
||||
handler.run(ctx)
|
||||
except RuntimeError:
|
||||
pass
|
||||
exit_.assert_called_once_with(1)
|
||||
|
||||
# runs with output_file_path
|
||||
with patch.object(sys, "exit") as exit_:
|
||||
exit_.side_effect = RuntimeError
|
||||
ctx = Object(
|
||||
params={"output_file_path": self.tempdir},
|
||||
parent=Object(
|
||||
params={},
|
||||
),
|
||||
)
|
||||
# self.assertRaises(FileNotFoundError, handler.run, ctx)
|
||||
handler.run(ctx)
|
||||
exit_.assert_not_called()
|
||||
|
||||
def test_list_models(self):
|
||||
handler = self.make_handler(
|
||||
|
|
@ -80,6 +166,23 @@ class TestImporterCommand(TestCase):
|
|||
self.assertIn("dry_run", sig2.parameters)
|
||||
|
||||
|
||||
class TestFileExporterCommand(TestCase):
|
||||
|
||||
def test_basic(self):
|
||||
def myfunc(ctx, **kwargs):
|
||||
pass
|
||||
|
||||
sig1 = inspect.signature(myfunc)
|
||||
self.assertIn("kwargs", sig1.parameters)
|
||||
self.assertNotIn("dry_run", sig1.parameters)
|
||||
self.assertNotIn("output_file_path", sig1.parameters)
|
||||
wrapt = mod.file_export_command(myfunc)
|
||||
sig2 = inspect.signature(wrapt)
|
||||
self.assertNotIn("kwargs", sig2.parameters)
|
||||
self.assertIn("dry_run", sig2.parameters)
|
||||
self.assertIn("output_file_path", sig2.parameters)
|
||||
|
||||
|
||||
class TestFileImporterCommand(TestCase):
|
||||
|
||||
def test_basic(self):
|
||||
|
|
|
|||
22
tests/cli/test_export_csv.py
Normal file
22
tests/cli/test_export_csv.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from unittest import TestCase
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from wuttasync.cli import export_csv as mod, ImportCommandHandler
|
||||
|
||||
|
||||
class TestExportCsv(TestCase):
|
||||
|
||||
def test_basic(self):
|
||||
params = {
|
||||
"models": [],
|
||||
"create": True,
|
||||
"update": True,
|
||||
"delete": False,
|
||||
"dry_run": True,
|
||||
}
|
||||
ctx = MagicMock(params=params)
|
||||
with patch.object(ImportCommandHandler, "run") as run:
|
||||
mod.export_csv(ctx)
|
||||
run.assert_called_once_with(ctx)
|
||||
|
|
@ -19,4 +19,4 @@ class TestImportCsv(TestCase):
|
|||
ctx = MagicMock(params=params)
|
||||
with patch.object(ImportCommandHandler, "run") as run:
|
||||
mod.import_csv(ctx)
|
||||
run.assert_called_once_with(params)
|
||||
run.assert_called_once_with(ctx)
|
||||
|
|
|
|||
|
|
@ -19,4 +19,4 @@ class TestImportCsv(TestCase):
|
|||
ctx = MagicMock(params=params)
|
||||
with patch.object(ImportCommandHandler, "run") as run:
|
||||
mod.import_versions(ctx)
|
||||
run.assert_called_once_with(params)
|
||||
run.assert_called_once_with(ctx)
|
||||
|
|
|
|||
99
tests/exporting/test_base.py
Normal file
99
tests/exporting/test_base.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from wuttjamaican.testing import DataTestCase
|
||||
|
||||
from wuttasync.exporting import base as mod, ExportHandler
|
||||
|
||||
|
||||
class TestToFile(DataTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.setup_db()
|
||||
self.handler = ExportHandler(self.config)
|
||||
|
||||
def make_exporter(self, **kwargs):
|
||||
kwargs.setdefault("handler", self.handler)
|
||||
return mod.ToFile(self.config, **kwargs)
|
||||
|
||||
def test_setup(self):
|
||||
model = self.app.model
|
||||
|
||||
# output file is opened
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
self.assertFalse(exp.dry_run)
|
||||
with patch.object(exp, "open_output_file") as open_output_file:
|
||||
exp.setup()
|
||||
open_output_file.assert_called_once_with()
|
||||
|
||||
# but not if in dry run mode
|
||||
with patch.object(self.handler, "dry_run", new=True):
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
self.assertTrue(exp.dry_run)
|
||||
with patch.object(exp, "open_output_file") as open_output_file:
|
||||
exp.setup()
|
||||
open_output_file.assert_not_called()
|
||||
|
||||
def test_teardown(self):
|
||||
model = self.app.model
|
||||
|
||||
# output file is closed
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
self.assertFalse(exp.dry_run)
|
||||
with patch.object(exp, "close_output_file") as close_output_file:
|
||||
exp.teardown()
|
||||
close_output_file.assert_called_once_with()
|
||||
|
||||
# but not if in dry run mode
|
||||
with patch.object(self.handler, "dry_run", new=True):
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
self.assertTrue(exp.dry_run)
|
||||
with patch.object(exp, "close_output_file") as close_output_file:
|
||||
exp.teardown()
|
||||
close_output_file.assert_not_called()
|
||||
|
||||
def test_get_output_file_path(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
|
||||
# output path must be set
|
||||
self.assertRaises(ValueError, exp.get_output_file_path)
|
||||
|
||||
# path is guessed from dir+filename
|
||||
path1 = self.write_file("data1.txt", "")
|
||||
exp.output_file_path = self.tempdir
|
||||
exp.output_file_name = "data1.txt"
|
||||
self.assertEqual(exp.get_output_file_path(), path1)
|
||||
|
||||
# path can be explicitly set
|
||||
path2 = self.write_file("data2.txt", "")
|
||||
exp.output_file_path = path2
|
||||
self.assertEqual(exp.get_output_file_path(), path2)
|
||||
|
||||
def test_get_output_file_name(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
|
||||
# name cannot be guessed
|
||||
self.assertRaises(NotImplementedError, exp.get_output_file_name)
|
||||
|
||||
# name can be explicitly set
|
||||
exp.output_file_name = "data.txt"
|
||||
self.assertEqual(exp.get_output_file_name(), "data.txt")
|
||||
|
||||
def test_open_output_file(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
self.assertRaises(NotImplementedError, exp.open_output_file)
|
||||
|
||||
def test_close_output_file(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
|
||||
path = self.write_file("data.txt", "")
|
||||
with open(path, "wt") as f:
|
||||
exp.output_file = f
|
||||
with patch.object(f, "close") as close:
|
||||
exp.close_output_file()
|
||||
close.assert_called_once_with()
|
||||
209
tests/exporting/test_csv.py
Normal file
209
tests/exporting/test_csv.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
import csv
|
||||
import io
|
||||
from unittest.mock import patch
|
||||
|
||||
from wuttjamaican.testing import DataTestCase
|
||||
|
||||
from wuttasync.exporting import csv as mod, ExportHandler
|
||||
from wuttasync.importing import FromWuttaHandler, FromWutta
|
||||
|
||||
|
||||
class TestToCsv(DataTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.setup_db()
|
||||
self.handler = ExportHandler(self.config)
|
||||
|
||||
def make_exporter(self, **kwargs):
|
||||
kwargs.setdefault("handler", self.handler)
|
||||
kwargs.setdefault("output_file_path", self.tempdir)
|
||||
return mod.ToCsv(self.config, **kwargs)
|
||||
|
||||
def test_get_output_file_name(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
|
||||
# name can be guessed
|
||||
self.assertEqual(exp.get_output_file_name(), "Setting.csv")
|
||||
|
||||
# name can be explicitly set
|
||||
exp.output_file_name = "data.txt"
|
||||
self.assertEqual(exp.get_output_file_name(), "data.txt")
|
||||
|
||||
def test_open_output_file(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
self.assertIsNone(exp.output_file)
|
||||
self.assertIsNone(exp.output_writer)
|
||||
exp.open_output_file()
|
||||
try:
|
||||
self.assertIsInstance(exp.output_file, io.TextIOBase)
|
||||
self.assertIsInstance(exp.output_writer, csv.DictWriter)
|
||||
finally:
|
||||
exp.output_file.close()
|
||||
|
||||
def test_close_output_file(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
|
||||
self.assertIsNone(exp.output_file)
|
||||
self.assertIsNone(exp.output_writer)
|
||||
exp.open_output_file()
|
||||
self.assertIsNotNone(exp.output_file)
|
||||
self.assertIsNotNone(exp.output_writer)
|
||||
exp.close_output_file()
|
||||
self.assertIsNone(exp.output_file)
|
||||
self.assertIsNone(exp.output_writer)
|
||||
|
||||
def test_coerce_csv(self):
|
||||
model = self.app.model
|
||||
|
||||
# string value
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
result = exp.coerce_csv({"name": "foo", "value": "bar"})
|
||||
self.assertEqual(result, {"name": "foo", "value": "bar"})
|
||||
|
||||
# null value converts to empty string
|
||||
result = exp.coerce_csv({"name": "foo", "value": None})
|
||||
self.assertEqual(result, {"name": "foo", "value": ""})
|
||||
|
||||
# float value passed thru as-is
|
||||
result = exp.coerce_csv({"name": "foo", "value": 12.34})
|
||||
self.assertEqual(result, {"name": "foo", "value": 12.34})
|
||||
self.assertIsInstance(result["value"], float)
|
||||
|
||||
def test_update_target_object(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(model_class=model.Setting)
|
||||
|
||||
exp.setup()
|
||||
|
||||
with patch.object(exp, "output_writer") as output_writer:
|
||||
|
||||
# writer is called for normal run
|
||||
data = {"name": "foo", "value": "bar"}
|
||||
exp.update_target_object(None, data)
|
||||
output_writer.writerow.assert_called_once_with(data)
|
||||
|
||||
# but not called for dry run
|
||||
output_writer.writerow.reset_mock()
|
||||
with patch.object(self.handler, "dry_run", new=True):
|
||||
exp.update_target_object(None, data)
|
||||
output_writer.writerow.assert_not_called()
|
||||
|
||||
exp.teardown()
|
||||
|
||||
|
||||
class MockMixinExporter(mod.FromSqlalchemyToCsvMixin, FromWutta, mod.ToCsv):
|
||||
pass
|
||||
|
||||
|
||||
class TestFromSqlalchemyToCsvMixin(DataTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.setup_db()
|
||||
self.handler = ExportHandler(self.config)
|
||||
|
||||
def make_exporter(self, **kwargs):
|
||||
kwargs.setdefault("handler", self.handler)
|
||||
return MockMixinExporter(self.config, **kwargs)
|
||||
|
||||
def test_model_title(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(source_model_class=model.Setting)
|
||||
|
||||
# default comes from model class
|
||||
self.assertEqual(exp.get_model_title(), "Setting")
|
||||
|
||||
# but can override
|
||||
exp.model_title = "Widget"
|
||||
self.assertEqual(exp.get_model_title(), "Widget")
|
||||
|
||||
def test_get_simple_fields(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(source_model_class=model.Setting)
|
||||
|
||||
# default comes from model class
|
||||
self.assertEqual(exp.get_simple_fields(), ["name", "value"])
|
||||
|
||||
# but can override
|
||||
exp.simple_fields = ["name"]
|
||||
self.assertEqual(exp.get_simple_fields(), ["name"])
|
||||
|
||||
# no default if no model class
|
||||
exp = self.make_exporter()
|
||||
self.assertEqual(exp.get_simple_fields(), [])
|
||||
|
||||
def test_normalize_source_object(self):
|
||||
model = self.app.model
|
||||
exp = self.make_exporter(source_model_class=model.Setting)
|
||||
setting = model.Setting(name="foo", value="bar")
|
||||
data = exp.normalize_source_object(setting)
|
||||
self.assertEqual(data, {"name": "foo", "value": "bar"})
|
||||
|
||||
def test_make_object(self):
|
||||
model = self.app.model
|
||||
|
||||
# normal
|
||||
exp = self.make_exporter(source_model_class=model.Setting)
|
||||
obj = exp.make_object()
|
||||
self.assertIsInstance(obj, model.Setting)
|
||||
|
||||
# no model_class
|
||||
exp = self.make_exporter()
|
||||
self.assertRaises(TypeError, exp.make_object)
|
||||
|
||||
|
||||
class MockMixinHandler(
|
||||
mod.FromSqlalchemyToCsvHandlerMixin, FromWuttaHandler, mod.ToCsvHandler
|
||||
):
|
||||
FromImporterBase = FromWutta
|
||||
|
||||
|
||||
class TestFromSqlalchemyToCsvHandlerMixin(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return MockMixinHandler(self.config, **kwargs)
|
||||
|
||||
def test_get_source_model(self):
|
||||
with patch.object(
|
||||
mod.FromSqlalchemyToCsvHandlerMixin, "define_importers", return_value={}
|
||||
):
|
||||
handler = self.make_handler()
|
||||
self.assertRaises(NotImplementedError, handler.get_source_model)
|
||||
|
||||
def test_define_importers(self):
|
||||
model = self.app.model
|
||||
with patch.object(
|
||||
mod.FromSqlalchemyToCsvHandlerMixin, "get_source_model", return_value=model
|
||||
):
|
||||
handler = self.make_handler()
|
||||
importers = handler.define_importers()
|
||||
self.assertIn("Setting", importers)
|
||||
self.assertTrue(issubclass(importers["Setting"], FromWutta))
|
||||
self.assertTrue(issubclass(importers["Setting"], mod.ToCsv))
|
||||
self.assertIn("User", importers)
|
||||
self.assertIn("Person", importers)
|
||||
self.assertIn("Role", importers)
|
||||
|
||||
def test_make_importer_factory(self):
|
||||
model = self.app.model
|
||||
with patch.object(
|
||||
mod.FromSqlalchemyToCsvHandlerMixin, "define_importers", return_value={}
|
||||
):
|
||||
handler = self.make_handler()
|
||||
factory = handler.make_importer_factory(model.Setting, "Setting")
|
||||
self.assertTrue(issubclass(factory, FromWutta))
|
||||
self.assertTrue(issubclass(factory, mod.ToCsv))
|
||||
|
||||
|
||||
class TestFromWuttaToCsv(DataTestCase):
|
||||
|
||||
def make_handler(self, **kwargs):
|
||||
return mod.FromWuttaToCsv(self.config, **kwargs)
|
||||
|
||||
def test_get_source_model(self):
|
||||
handler = self.make_handler()
|
||||
self.assertIs(handler.get_source_model(), self.app.model)
|
||||
4
tests/exporting/test_handlers.py
Normal file
4
tests/exporting/test_handlers.py
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
# nothing to test yet really, just ensuring coverage
|
||||
from wuttasync.exporting import handlers as mod
|
||||
|
|
@ -271,6 +271,9 @@ class Example(Base):
|
|||
flag = sa.Column(sa.Boolean(), nullable=False)
|
||||
optional_flag = sa.Column(sa.Boolean(), nullable=True)
|
||||
|
||||
date = sa.Column(sa.Date(), nullable=False)
|
||||
optional_date = sa.Column(sa.Date(), nullable=True)
|
||||
|
||||
dt = sa.Column(sa.DateTime(), nullable=False)
|
||||
optional_dt = sa.Column(sa.DateTime(), nullable=True)
|
||||
|
||||
|
|
@ -285,7 +288,7 @@ class TestMakeCoercers(TestCase):
|
|||
|
||||
def test_basic(self):
|
||||
coercers = mod.make_coercers(Example)
|
||||
self.assertEqual(len(coercers), 12)
|
||||
self.assertEqual(len(coercers), 14)
|
||||
|
||||
self.assertIs(coercers["id"], mod.coerce_integer)
|
||||
self.assertIs(coercers["optional_id"], mod.coerce_integer)
|
||||
|
|
@ -293,6 +296,8 @@ class TestMakeCoercers(TestCase):
|
|||
self.assertIs(coercers["optional_name"], mod.coerce_string_nullable)
|
||||
self.assertIs(coercers["flag"], mod.coerce_boolean)
|
||||
self.assertIs(coercers["optional_flag"], mod.coerce_boolean_nullable)
|
||||
self.assertIs(coercers["date"], mod.coerce_date)
|
||||
self.assertIs(coercers["optional_date"], mod.coerce_date)
|
||||
self.assertIs(coercers["dt"], mod.coerce_datetime)
|
||||
self.assertIs(coercers["optional_dt"], mod.coerce_datetime)
|
||||
self.assertIs(coercers["dec"], mod.coerce_decimal)
|
||||
|
|
@ -322,6 +327,12 @@ class TestMakeCoercer(TestCase):
|
|||
func = mod.make_coercer(Example.optional_flag)
|
||||
self.assertIs(func, mod.coerce_boolean_nullable)
|
||||
|
||||
func = mod.make_coercer(Example.date)
|
||||
self.assertIs(func, mod.coerce_date)
|
||||
|
||||
func = mod.make_coercer(Example.optional_date)
|
||||
self.assertIs(func, mod.coerce_date)
|
||||
|
||||
func = mod.make_coercer(Example.dt)
|
||||
self.assertIs(func, mod.coerce_datetime)
|
||||
|
||||
|
|
@ -365,6 +376,15 @@ class TestCoercers(TestCase):
|
|||
|
||||
self.assertIsNone(mod.coerce_boolean_nullable(""))
|
||||
|
||||
def test_coerce_date(self):
|
||||
self.assertIsNone(mod.coerce_date(""))
|
||||
|
||||
value = mod.coerce_date("2025-10-19")
|
||||
self.assertIsInstance(value, datetime.date)
|
||||
self.assertEqual(value, datetime.date(2025, 10, 19))
|
||||
|
||||
self.assertRaises(ValueError, mod.coerce_date, "XXX")
|
||||
|
||||
def test_coerce_datetime(self):
|
||||
self.assertIsNone(mod.coerce_datetime(""))
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,13 @@ class TestImportHandler(DataTestCase):
|
|||
handler.target_title = "Wutta"
|
||||
self.assertEqual(str(handler), "CSV → Wutta")
|
||||
|
||||
def test_actioner(self):
|
||||
handler = self.make_handler()
|
||||
self.assertEqual(handler.actioner, "importer")
|
||||
|
||||
handler.orientation = mod.Orientation.EXPORT
|
||||
self.assertEqual(handler.actioner, "exporter")
|
||||
|
||||
def test_actioning(self):
|
||||
handler = self.make_handler()
|
||||
self.assertEqual(handler.actioning, "importing")
|
||||
|
|
@ -190,6 +197,22 @@ class TestImportHandler(DataTestCase):
|
|||
self.assertNotIn("warnings_max_diffs", kw)
|
||||
self.assertEqual(handler.warnings_max_diffs, 30)
|
||||
|
||||
# runas_username (consumed)
|
||||
self.assertIsNone(handler.runas_username)
|
||||
kw["runas_username"] = "fred"
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertNotIn("runas_username", kw)
|
||||
self.assertEqual(handler.runas_username, "fred")
|
||||
|
||||
# transaction_comment (consumed)
|
||||
self.assertIsNone(handler.transaction_comment)
|
||||
kw["transaction_comment"] = "hello world"
|
||||
result = handler.consume_kwargs(kw)
|
||||
self.assertIs(result, kw)
|
||||
self.assertNotIn("transaction_comment", kw)
|
||||
self.assertEqual(handler.transaction_comment, "hello world")
|
||||
|
||||
def test_define_importers(self):
|
||||
handler = self.make_handler()
|
||||
importers = handler.define_importers()
|
||||
|
|
@ -490,11 +513,41 @@ class TestToWuttaHandler(DataTestCase):
|
|||
self.assertEqual(handler.get_target_title(), "what_about_this")
|
||||
|
||||
def test_make_target_session(self):
|
||||
model = self.app.model
|
||||
handler = self.make_handler()
|
||||
|
||||
# makes "new" (mocked in our case) app session
|
||||
fred = model.User(username="fred")
|
||||
self.session.add(fred)
|
||||
self.session.commit()
|
||||
|
||||
# makes "new" (mocked in our case) app session, with no runas
|
||||
# username set by default
|
||||
with patch.object(self.app, "make_session") as make_session:
|
||||
make_session.return_value = self.session
|
||||
session = handler.make_target_session()
|
||||
make_session.assert_called_once_with()
|
||||
self.assertIs(session, self.session)
|
||||
self.assertNotIn("continuum_user_id", session.info)
|
||||
self.assertNotIn("continuum_user_id", self.session.info)
|
||||
|
||||
# runas user also should not be set, if username is not valid
|
||||
handler.runas_username = "freddie"
|
||||
with patch.object(self.app, "make_session") as make_session:
|
||||
make_session.return_value = self.session
|
||||
session = handler.make_target_session()
|
||||
make_session.assert_called_once_with()
|
||||
self.assertIs(session, self.session)
|
||||
self.assertNotIn("continuum_user_id", session.info)
|
||||
self.assertNotIn("continuum_user_id", self.session.info)
|
||||
|
||||
# this time we should have runas user properly set
|
||||
handler.runas_username = "fred"
|
||||
with patch.object(self.app, "make_session") as make_session:
|
||||
make_session.return_value = self.session
|
||||
session = handler.make_target_session()
|
||||
make_session.assert_called_once_with()
|
||||
self.assertIs(session, self.session)
|
||||
self.assertIn("continuum_user_id", session.info)
|
||||
self.assertEqual(session.info["continuum_user_id"], fred.uuid)
|
||||
self.assertIn("continuum_user_id", self.session.info)
|
||||
self.assertEqual(self.session.info["continuum_user_id"], fred.uuid)
|
||||
|
|
|
|||
|
|
@ -14,20 +14,6 @@ class TestFromWuttaToVersions(VersionTestCase):
|
|||
def make_handler(self, **kwargs):
|
||||
return mod.FromWuttaToVersions(self.config, **kwargs)
|
||||
|
||||
def test_consume_kwargs(self):
|
||||
|
||||
# no comment by default
|
||||
handler = self.make_handler()
|
||||
kw = handler.consume_kwargs({})
|
||||
self.assertEqual(kw, {})
|
||||
self.assertIsNone(handler.continuum_comment)
|
||||
|
||||
# but can provide one
|
||||
handler = self.make_handler()
|
||||
kw = handler.consume_kwargs({"comment": "yeehaw"})
|
||||
self.assertEqual(kw, {})
|
||||
self.assertEqual(handler.continuum_comment, "yeehaw")
|
||||
|
||||
def test_begin_target_transaction(self):
|
||||
model = self.app.model
|
||||
txncls = continuum.transaction_class(model.User)
|
||||
|
|
@ -44,7 +30,7 @@ class TestFromWuttaToVersions(VersionTestCase):
|
|||
|
||||
# with comment
|
||||
handler = self.make_handler()
|
||||
handler.continuum_comment = "yeehaw"
|
||||
handler.transaction_comment = "yeehaw"
|
||||
handler.begin_target_transaction()
|
||||
self.assertIn("comment", handler.continuum_txn.meta)
|
||||
self.assertEqual(handler.continuum_txn.meta["comment"], "yeehaw")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
# -*- coding: utf-8; -*-
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from wuttjamaican.testing import ConfigTestCase
|
||||
|
||||
from wuttasync import app as mod
|
||||
|
|
@ -16,6 +18,16 @@ class FromCsvToPoser(FromCsvToWutta):
|
|||
pass
|
||||
|
||||
|
||||
class FromFooToBaz1(ImportHandler):
|
||||
source_key = "foo"
|
||||
target_key = "baz"
|
||||
|
||||
|
||||
class FromFooToBaz2(ImportHandler):
|
||||
source_key = "foo"
|
||||
target_key = "baz"
|
||||
|
||||
|
||||
class TestWuttaSyncAppProvider(ConfigTestCase):
|
||||
|
||||
def test_get_all_import_handlers(self):
|
||||
|
|
@ -100,6 +112,18 @@ class TestWuttaSyncAppProvider(ConfigTestCase):
|
|||
any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
|
||||
)
|
||||
|
||||
# nothing returned if multiple handlers found but none are designated
|
||||
with patch.object(
|
||||
self.app.providers["wuttasync"],
|
||||
"get_all_import_handlers",
|
||||
return_value=[FromFooToBaz1, FromFooToBaz2],
|
||||
):
|
||||
handlers = self.app.get_designated_import_handlers()
|
||||
baz_handlers = [
|
||||
h for h in handlers if h.get_key() == "import.to_baz.from_foo"
|
||||
]
|
||||
self.assertEqual(len(baz_handlers), 0)
|
||||
|
||||
def test_get_import_handler(self):
|
||||
|
||||
# make sure a basic fetch works
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue