Compare commits

...

12 commits

35 changed files with 1462 additions and 75 deletions

View file

@ -5,6 +5,28 @@ All notable changes to WuttaSync will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
## v0.5.0 (2026-01-03)
### Feat
- add support for `wutta export-csv` command
### Fix
- add `actioner` property for ImportHandler
## v0.4.0 (2025-12-31)
### Feat
- add support for `--comment` CLI param, to set versioning comment
- add support for `--runas` CLI param, to set versioning authorship
### Fix
- make pylint happy
- accept either `--recip` or `--recips` param for import commands
## v0.3.0 (2025-12-20)
### Feat

View file

@ -0,0 +1,6 @@
``wuttasync.cli.export_csv``
============================
.. automodule:: wuttasync.cli.export_csv
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting.base``
============================
.. automodule:: wuttasync.exporting.base
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting.csv``
===========================
.. automodule:: wuttasync.exporting.csv
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting.handlers``
================================
.. automodule:: wuttasync.exporting.handlers
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting``
=======================
.. automodule:: wuttasync.exporting
:members:

View file

@ -73,9 +73,14 @@ cf. :doc:`rattail-manual:data/sync/index`.
api/wuttasync.app
api/wuttasync.cli
api/wuttasync.cli.base
api/wuttasync.cli.export_csv
api/wuttasync.cli.import_csv
api/wuttasync.cli.import_versions
api/wuttasync.emails
api/wuttasync.exporting
api/wuttasync.exporting.base
api/wuttasync.exporting.csv
api/wuttasync.exporting.handlers
api/wuttasync.importing
api/wuttasync.importing.base
api/wuttasync.importing.csv

View file

@ -9,6 +9,24 @@ WuttaSync.
It is fairly simple to add more; see :doc:`custom`.
.. _wutta-export-csv:
``wutta export-csv``
--------------------
Export data from the Wutta :term:`app database` to CSV file(s).
This *should* be able to automatically export any table mapped in the
:term:`app model`. The only caveat is that it is "dumb" and does not
have any special field handling. This means the column headers in the
CSV will be the same as in the source table, and some data types may
not behave as expected etc.
Defined in: :mod:`wuttasync.cli.export_csv`
.. program-output:: wutta export-csv --help
.. _wutta-import-csv:
``wutta import-csv``

View file

@ -38,7 +38,7 @@ Here is the code and we'll explain below::
config = ctx.parent.wutta_config
handler = ImportCommandHandler(
config, import_handler='poser.importing.foo:FromFooToPoser')
handler.run(ctx.params)
handler.run(ctx)
Hopefully it's straightforward but to be clear:

View file

@ -6,7 +6,7 @@ build-backend = "hatchling.build"
[project]
name = "WuttaSync"
version = "0.3.0"
version = "0.5.0"
description = "Wutta Framework for data import/export and real-time sync"
readme = "README.md"
authors = [{name = "Lance Edgar", email = "lance@wuttaproject.org"}]
@ -28,8 +28,9 @@ requires-python = ">= 3.8"
dependencies = [
"humanize",
"makefun",
"rich",
"SQLAlchemy-Utils",
"WuttJamaican[db]>=0.27.0",
"WuttJamaican[db]>=0.28.1",
]
@ -42,6 +43,7 @@ tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"]
wuttasync = "wuttasync.app:WuttaSyncAppProvider"
[project.entry-points."wuttasync.importing"]
"export.to_csv.from_wutta" = "wuttasync.exporting.csv:FromWuttaToCsv"
"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions"
"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta"

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -26,12 +26,19 @@ WuttaSync - ``wutta`` subcommands
This namespace exposes the following:
* :func:`~wuttasync.cli.base.import_command()`
* :func:`~wuttasync.cli.base.file_export_command()`
* :func:`~wuttasync.cli.base.file_import_command()`
* :class:`~wuttasync.cli.base.ImportCommandHandler`
"""
from .base import import_command, file_import_command, ImportCommandHandler
from .base import (
import_command,
file_export_command,
file_import_command,
ImportCommandHandler,
)
# nb. must bring in all modules for discovery to work
from . import export_csv
from . import import_csv
from . import import_versions

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -32,10 +32,12 @@ from typing import List, Optional
from typing_extensions import Annotated
import makefun
import rich
import typer
from wuttjamaican.app import GenericHandler
from wuttasync.importing import ImportHandler
from wuttasync.importing import ImportHandler, FromFileHandler
from wuttasync.exporting import ToFileHandler
log = logging.getLogger(__name__)
@ -102,7 +104,7 @@ class ImportCommandHandler(GenericHandler):
elif key:
self.import_handler = self.app.get_import_handler(key, require=True)
def run(self, params, progress=None): # pylint: disable=unused-argument
def run(self, ctx, progress=None): # pylint: disable=unused-argument
"""
Run the import/export job(s) based on command line params.
@ -113,21 +115,51 @@ class ImportCommandHandler(GenericHandler):
Unless ``--list-models`` was specified on the command line in
which case we do :meth:`list_models()` instead.
:param params: Dict of params from command line. This must
include a ``'models'`` key, the rest are optional.
:param ctx: :class:`typer.Context` instance.
:param progress: Optional progress indicator factory.
"""
# maybe just list models and bail
if params.get("list_models"):
self.list_models(params)
if ctx.params.get("list_models"):
self.list_models(ctx.params)
return
# otherwise process some data
# otherwise we'll (hopefully) process some data
log.debug("using handler: %s", self.import_handler.get_spec())
kw = dict(params)
models = kw.pop("models")
# but first, some extra checks for certain file-based
# handlers. this must be done here, because these CLI params
# are not technically required (otherwise typer would handle
# this instead of us here). and that is because we want to
# allow user to specify --list without needing to also specify
# --input or --output
if isinstance(self.import_handler, FromFileHandler):
if not ctx.params.get("input_file_path"):
rich.print(
"\n[bold yellow]must specify --input folder/file path[/bold yellow]\n",
file=sys.stderr,
)
sys.exit(1)
elif isinstance(self.import_handler, ToFileHandler):
if not ctx.params.get("output_file_path"):
rich.print(
"\n[bold yellow]must specify --output folder/file path[/bold yellow]\n",
file=sys.stderr,
)
sys.exit(1)
# all params from caller will be passed along
kw = dict(ctx.params)
# runas user and comment also, but they come from root command
if username := ctx.parent.params.get("runas_username"):
kw["runas_username"] = username
if comment := ctx.parent.params.get("comment"):
kw["transaction_comment"] = comment
# sort out which models to process
models = kw.pop("models", None)
if not models:
models = list(self.import_handler.importers)
log.debug(
@ -136,6 +168,8 @@ class ImportCommandHandler(GenericHandler):
self.import_handler.get_title(),
", ".join(models),
)
# process data
log.debug("params are: %s", kw)
self.import_handler.process_data(*models, **kw)
@ -159,7 +193,7 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
models: Annotated[
Optional[List[str]],
typer.Argument(
help="Model(s) to process. Can specify one or more, "
help="Target model(s) to process. Specify one or more, "
"or omit to process default models."
),
] = None,
@ -248,7 +282,9 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
warnings_recipients: Annotated[
str,
typer.Option(
"--recip", help="Override the recipient(s) for diff warning email."
"--recip",
"--recips",
help="Override the recipient(s) for diff warning email.",
),
] = None,
warnings_max_diffs: Annotated[
@ -311,17 +347,65 @@ def import_command(fn):
return makefun.create_function(final_sig, fn)
def file_import_command_template( # pylint: disable=unused-argument
input_file_path: Annotated[
def file_export_command_template( # pylint: disable=unused-argument
# nb. technically this is required, but not if doing --list
# (so we cannot mark it required here, for that reason)
output_file_path: Annotated[
Path,
typer.Option(
"--input-path",
"--output",
"-o",
exists=True,
file_okay=True,
dir_okay=True,
help="Path to input file(s). Can be a folder "
"if app logic can guess the filename(s); "
"otherwise must be complete file path.",
help="Path to output folder. Or full path to output file "
"if only running one target model.",
),
] = None,
):
"""
Stub function to provide signature for exporter commands which
produce data file(s) as output. Used with
:func:`file_export_command`.
"""
def file_export_command(fn):
"""
Decorator for file export commands. Adds common params based on
:func:`file_export_command_template`.
"""
original_sig = inspect.signature(fn)
plain_import_sig = inspect.signature(import_command_template)
file_export_sig = inspect.signature(file_export_command_template)
desired_params = list(plain_import_sig.parameters.values()) + list(
file_export_sig.parameters.values()
)
params = list(original_sig.parameters.values())
for i, param in enumerate(desired_params):
params.insert(i + 1, param)
# remove the **kwargs param
params.pop(-1)
final_sig = original_sig.replace(parameters=params)
return makefun.create_function(final_sig, fn)
def file_import_command_template( # pylint: disable=unused-argument
# nb. technically this is required, but not if doing --list
# (so we cannot mark it required here, for that reason)
input_file_path: Annotated[
Path,
typer.Option(
"--input",
"-i",
exists=True,
file_okay=True,
dir_okay=True,
help="Path to input folder. Or full path to input file "
"if only running one target model.",
),
] = None,
):

View file

@ -0,0 +1,42 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
See also: :ref:`wutta-export-csv`
"""
import typer
from wuttjamaican.cli import wutta_typer
from .base import file_export_command, ImportCommandHandler
@wutta_typer.command()
@file_export_command
def export_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
"""
Export data from Wutta DB to CSV file(s)
"""
config = ctx.parent.wutta_config
handler = ImportCommandHandler(config, key="export.to_csv.from_wutta")
handler.run(ctx)

View file

@ -39,4 +39,4 @@ def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
"""
config = ctx.parent.wutta_config
handler = ImportCommandHandler(config, key="import.to_wutta.from_csv")
handler.run(ctx.params)
handler.run(ctx)

View file

@ -28,7 +28,6 @@ import sys
import rich
import typer
from typing_extensions import Annotated
from wuttjamaican.cli import wutta_typer
@ -37,14 +36,7 @@ from .base import import_command, ImportCommandHandler
@wutta_typer.command()
@import_command
def import_versions( # pylint: disable=unused-argument
ctx: typer.Context,
comment: Annotated[
str,
typer.Option("--comment", "-m", help="Comment to set on the transaction."),
] = "import catch-up versions",
**kwargs,
):
def import_versions(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
"""
Import latest data to version tables, for Wutta DB
"""
@ -70,4 +62,4 @@ def import_versions( # pylint: disable=unused-argument
sys.exit(1)
handler = ImportCommandHandler(config, key="import.to_versions.from_wutta")
handler.run(ctx.params)
handler.run(ctx)

View file

@ -0,0 +1,43 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Data Import / Export Framework
This namespace exposes the following:
* :enum:`~wuttasync.importing.handlers.Orientation`
And some :term:`export handler <import handler>` base classes:
* :class:`~wuttasync.exporting.handlers.ExportHandler`
* :class:`~wuttasync.exporting.handlers.ToFileHandler`
And some :term:`exporter <importer>` base classes:
* :class:`~wuttasync.exporting.base.ToFile`
See also the :mod:`wuttasync.importing` module.
"""
from .handlers import Orientation, ExportHandler, ToFileHandler
from .base import ToFile

View file

@ -0,0 +1,166 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Data exporter base classes
"""
import os
from wuttasync.importing import Importer
class ToFile(Importer):
"""
Base class for importer/exporter using output file as data target.
Depending on the subclass, it may be able to "guess" (at least
partially) the path to the output file. If not, and/or to avoid
ambiguity, the caller must specify the file path.
In most cases caller may specify any of these via kwarg to the
class constructor, or e.g.
:meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`:
* :attr:`output_file_path`
* :attr:`output_file_name`
The subclass itself can also specify via override of these
methods:
* :meth:`get_output_file_path()`
* :meth:`get_output_file_name()`
And of course subclass must override these too:
* :meth:`open_output_file()`
* :meth:`close_output_file()`
* (and see also :attr:`output_file`)
"""
output_file_path = None
"""
Path to output folder, or file.
The ideal usage is to set this to the output *folder* path. That
allows the handler to run several importers in one go. The same
output folder path is given to each importer; they then each
determine their own output filename within that.
But you can also set this to the full output folder + file path,
e.g. if you're just running one importer. This would override
the importer's own logic for determining output filename.
See also :meth:`get_output_file_path()` and
:meth:`get_output_file_name()`.
"""
output_file_name = None
"""
Optional static output file name (sans folder path).
If set, this will be used as output filename instead of the
importer determining one on its own.
See also :meth:`get_output_file_name()`.
"""
output_file = None
"""
Handle to the open output file, if applicable. May be set by
:meth:`open_output_file()` for later reference within
:meth:`close_output_file()`.
"""
def setup(self):
"""
Open the output file. See also :meth:`open_output_file()`.
"""
if not self.dry_run:
self.open_output_file()
def teardown(self):
"""
Close the output file. See also :meth:`close_output_file()`.
"""
if not self.dry_run:
self.close_output_file()
def get_output_file_path(self):
"""
This must return the full path to output file.
Default logic inspects :attr:`output_file_path`; if that
points to a folder then it is combined with
:meth:`get_output_file_name()`. Otherwise it's returned
as-is.
:returns: Path to output file, as string
"""
path = self.output_file_path
if not path:
raise ValueError("must set output_file_path")
if os.path.isdir(path):
filename = self.get_output_file_name()
return os.path.join(path, filename)
return path
def get_output_file_name(self):
"""
This must return the output filename, sans folder path.
Default logic will return :attr:`output_file_name` if set,
otherwise raise error.
:returns: Output filename, sans folder path
"""
if self.output_file_name:
return self.output_file_name
raise NotImplementedError("can't guess output filename")
def open_output_file(self):
"""
Open the output file for writing target data.
Subclass must override to specify how this happens; default
logic is not implemented. Remember to set :attr:`output_file`
if applicable for reference when closing.
See also :attr:`get_output_file_path()` and
:meth:`close_output_file()`.
"""
raise NotImplementedError
def close_output_file(self):
"""
Close the output file for target data.
Subclass must override to specify how this happens; default
logic blindly calls the ``close()`` method on whatever
:attr:`output_file` happens to point to.
See also :attr:`open_output_file()`.
"""
self.output_file.close()

View file

@ -0,0 +1,321 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Exporting to CSV
"""
import csv
import logging
from collections import OrderedDict
import sqlalchemy as sa
from sqlalchemy_utils.functions import get_primary_keys, get_columns
from wuttjamaican.db.util import make_topo_sortkey
from wuttasync.importing import FromWuttaHandler, FromWutta
from wuttasync.exporting import ToFileHandler, ToFile
log = logging.getLogger(__name__)
class ToCsv(ToFile): # pylint: disable=abstract-method
"""
Base class for exporter using CSV file as data target.
This inherits from :class:`~wuttasync.exporting.base.ToFile`.
"""
output_writer = None
"""
While the output file is open, this will reference a
:class:`python:csv.DictWriter` instance.
"""
csv_encoding = "utf_8"
"""
Encoding used for the CSV output file.
You can specify an override if needed when calling
:meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`.
"""
def get_output_file_name(self): # pylint: disable=empty-docstring
""" """
if self.output_file_name:
return self.output_file_name
model_title = self.get_model_title()
return f"{model_title}.csv"
def open_output_file(self):
"""
Opens the output CSV file for writing.
This calls
:meth:`~wuttasync.exporting.base.ToFile.get_output_file_path()`
and opens that file. It sets
:attr:`~wuttasync.exporting.base.ToFile.output_file` and also
:attr:`output_writer`. And it calls
:meth:`write_output_header()` to write the field header row.
"""
path = self.get_output_file_path()
log.debug("opening output file: %s", path)
self.output_file = open( # pylint: disable=consider-using-with
path, "wt", encoding=self.csv_encoding
)
self.output_writer = csv.DictWriter(
self.output_file,
self.fields,
# quoting=csv.QUOTE_NONNUMERIC
)
self.write_output_header()
def write_output_header(self):
"""
Write the field header row to the CSV file.
Default logic calls
:meth:`~python:csv.DictWriter.writeheader()` on the
:attr:`output_writer` instance.
"""
self.output_writer.writeheader()
def close_output_file(self): # pylint: disable=empty-docstring
""" """
self.output_writer = None
self.output_file.close()
self.output_file = None
def update_target_object(self, obj, source_data, target_data=None):
"""
In a CSV export the assumption is we always start with an
empty file, so "create" is the only logical action for each
record - there are no updates or deletes per se.
But under the hood, this method is used for create as well, so
we override it and actually write the record to CSV file.
Unless :attr:`~wuttasync.importing.base.Importer.dry_run` is
true, this calls :meth:`~python:csv.csvwriter.writerow()` on
the :attr:`output_writer` instance.
See also parent method docs,
:meth:`~wuttasync.importing.base.Importer.update_target_object()`
"""
data = self.coerce_csv(source_data)
if not self.dry_run:
self.output_writer.writerow(data)
return data
def coerce_csv(self, data): # pylint: disable=missing-function-docstring
coerced = {}
for field in self.fields:
value = data[field]
if value is None:
value = ""
elif isinstance(value, (int, float)):
pass
else:
value = str(value)
coerced[field] = value
return coerced
class FromSqlalchemyToCsvMixin:
"""
Mixin class for SQLAlchemy ORM CSV :term:`exporters <importer>`.
Such exporters are generated automatically by
:class:`FromSqlalchemyToCsvHandlerMixin`, so you won't typically
reference this mixin class directly.
This mixin effectively behaves like the
:attr:`~wuttasync.importing.base.Importer.model_class` represents
the source side instead of the target. It uses
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class`
instead, for automatic things like inspecting the fields list.
"""
def get_model_title(self): # pylint: disable=missing-function-docstring
if hasattr(self, "model_title"):
return self.model_title
return self.source_model_class.__name__
def get_simple_fields(self): # pylint: disable=missing-function-docstring
if hasattr(self, "simple_fields"):
return self.simple_fields
try:
fields = get_columns(self.source_model_class)
except sa.exc.NoInspectionAvailable:
return []
return list(fields.keys())
def normalize_source_object(
self, obj
): # pylint: disable=missing-function-docstring
fields = self.get_fields()
fields = [f for f in self.get_simple_fields() if f in fields]
data = {field: getattr(obj, field) for field in fields}
return data
def make_object(self): # pylint: disable=missing-function-docstring
return self.source_model_class()
class FromSqlalchemyToCsvHandlerMixin:
"""
Mixin class for SQLAlchemy ORM CSV :term:`export handlers
<import handler>`.
This knows how to dynamically generate :term:`exporter <importer>`
classes to represent the models in the source ORM. Such classes
will inherit from :class:`FromSqlalchemyToCsvMixin`, in addition
to whatever :attr:`FromImporterBase` and :attr:`ToImporterBase`
reference.
That all happens within :meth:`define_importers()`.
"""
target_key = "csv"
generic_target_title = "CSV"
# nb. subclass must define this
FromImporterBase = None
"""
For a handler to use this mixin, it must set this to a valid base
class for the ORM source side. The :meth:`define_importers()`
logic will use this when dynamically generating new exporter
classes.
"""
ToImporterBase = ToCsv
"""
This must be set to a valid base class for the CSV target side.
Default is :class:`ToCsv` which should typically be fine; you can
change if needed.
"""
def get_source_model(self):
"""
This should return the :term:`app model` or a similar module
containing data model classes for the source side.
The source model is used to dynamically generate a set of
exporters (e.g. one per table in the source DB) which can use
CSV file as data target. See also :meth:`define_importers()`.
Subclass must override this if needed; default behavior is not
implemented.
"""
raise NotImplementedError
def define_importers(self):
"""
This mixin overrides typical (manual) importer definition, and
instead dynamically generates a set of exporters, e.g. one per
table in the source DB.
It does this based on the source model, as returned by
:meth:`get_source_model()`. It calls
:meth:`make_importer_factory()` for each model class found.
"""
importers = {}
model = self.get_source_model()
# mostly try to make an importer for every data model
for name in dir(model):
cls = getattr(model, name)
if (
isinstance(cls, type)
and issubclass(cls, model.Base)
and cls is not model.Base
):
importers[name] = self.make_importer_factory(cls, name)
# sort importers according to schema topography
topo_sortkey = make_topo_sortkey(model)
importers = OrderedDict(
[(name, importers[name]) for name in sorted(importers, key=topo_sortkey)]
)
return importers
def make_importer_factory(self, model_class, name):
"""
Generate a new :term:`exporter <importer>` class, targeting
the given :term:`data model` class.
The newly-created class will inherit from:
* :class:`FromSqlalchemyToCsvMixin`
* :attr:`FromImporterBase`
* :attr:`ToImporterBase`
:param model_class: A data model class.
:param name: The "model name" for the importer/exporter. New
class name will be based on this, so e.g. ``Widget`` model
name becomes ``WidgetImporter`` class name.
:returns: The new class, meant to process import/export
targeting the given data model.
"""
return type(
f"{name}Importer",
(FromSqlalchemyToCsvMixin, self.FromImporterBase, self.ToImporterBase),
{
"source_model_class": model_class,
"default_keys": list(get_primary_keys(model_class)),
},
)
class ToCsvHandler(ToFileHandler):
"""
Base class for export handlers using CSV file(s) as data target.
"""
class FromWuttaToCsv(
FromSqlalchemyToCsvHandlerMixin, FromWuttaHandler, ToCsvHandler
): # pylint: disable=too-many-ancestors
"""
Handler for Wutta (:term:`app database`) CSV export.
This uses :class:`FromSqlalchemyToCsvHandlerMixin` for most of the
heavy lifting.
"""
FromImporterBase = FromWutta
def get_source_model(self): # pylint: disable=empty-docstring
""" """
return self.app.model

View file

@ -0,0 +1,50 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Export Handlers
"""
from wuttasync.importing import ImportHandler, Orientation
class ExportHandler(ImportHandler):
"""
Generic base class for :term:`export handlers <import handler>`.
This is really just
:class:`~wuttasync.importing.handlers.ImportHandler` with the
orientation flipped.
"""
orientation = Orientation.EXPORT
"" # nb. suppress docs
class ToFileHandler(ExportHandler):
"""
Base class for export handlers which use output file(s) as the
data target.
Importers (exporters) used by this handler are generally assumed
to subclass :class:`~wuttasync.exporting.base.ToFile`.
"""

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -31,6 +31,8 @@ And some :term:`import handler` base classes:
* :class:`~wuttasync.importing.handlers.ImportHandler`
* :class:`~wuttasync.importing.handlers.FromFileHandler`
* :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`
* :class:`~wuttasync.importing.handlers.FromWuttaHandler`
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
* :class:`~wuttasync.importing.handlers.ToWuttaHandler`
@ -38,16 +40,22 @@ And some :term:`importer` base classes:
* :class:`~wuttasync.importing.base.Importer`
* :class:`~wuttasync.importing.base.FromFile`
* :class:`~wuttasync.importing.base.FromSqlalchemy`
* :class:`~wuttasync.importing.base.FromWutta`
* :class:`~wuttasync.importing.base.ToSqlalchemy`
* :class:`~wuttasync.importing.model.ToWutta`
See also the :mod:`wuttasync.exporting` module.
"""
from .handlers import (
Orientation,
ImportHandler,
FromFileHandler,
FromSqlalchemyHandler,
FromWuttaHandler,
ToSqlalchemyHandler,
ToWuttaHandler,
)
from .base import Importer, FromFile, ToSqlalchemy
from .base import Importer, FromFile, FromSqlalchemy, FromWutta, ToSqlalchemy
from .model import ToWutta

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -1448,6 +1448,13 @@ class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method
return self.normalize_target_object(obj)
class FromWutta(FromSqlalchemy): # pylint: disable=abstract-method
"""
Base class for data importer/exporter which uses the Wutta ORM
(:term:`app database`) as data source.
"""
class ToSqlalchemy(Importer):
"""
Base class for importer/exporter which uses SQLAlchemy ORM on the

View file

@ -391,6 +391,10 @@ def make_coercer(attr): # pylint: disable=too-many-return-statements
):
return coerce_datetime
# Date
if isinstance(attr.type, sa.Date):
return coerce_date
# Float
# nb. check this before decimal, since Numeric inherits from Float
if isinstance(attr.type, sa.Float):
@ -423,6 +427,13 @@ def coerce_boolean_nullable(value): # pylint: disable=missing-function-docstrin
return coerce_boolean(value)
def coerce_date(value): # pylint: disable=missing-function-docstring
if value == "":
return None
return datetime.datetime.strptime(value, "%Y-%m-%d").date()
def coerce_datetime(value): # pylint: disable=missing-function-docstring
if value == "":
return None

View file

@ -23,6 +23,7 @@
"""
Data Import / Export Handlers
"""
# pylint: disable=too-many-lines
import logging
import os
@ -48,7 +49,9 @@ class Orientation(Enum):
EXPORT = "export"
class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
class ImportHandler( # pylint: disable=too-many-public-methods,too-many-instance-attributes
GenericHandler
):
"""
Base class for all import/export handlers.
@ -166,6 +169,18 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
See also :attr:`warnings`.
"""
runas_username = None
"""
Username responsible for running the import/export job. This is
mostly used for Continuum versioning.
"""
transaction_comment = None
"""
Optional comment to apply to the transaction, where applicable.
This is mostly used for Continuum versioning.
"""
importers = None
"""
This should be a dict of all importer/exporter classes available
@ -195,6 +210,19 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
""" """
return self.get_title()
@property
def actioner(self):
"""
Convenience property which effectively returns the
:attr:`orientation` as a noun - i.e. one of:
* ``'importer'``
* ``'exporter'``
See also :attr:`actioning`.
"""
return f"{self.orientation.value}er"
@property
def actioning(self):
"""
@ -203,6 +231,8 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
* ``'importing'``
* ``'exporting'``
See also :attr:`actioner`.
"""
return f"{self.orientation.value}ing"
@ -416,6 +446,12 @@ class ImportHandler(GenericHandler): # pylint: disable=too-many-public-methods
if "warnings_max_diffs" in kwargs:
self.warnings_max_diffs = kwargs.pop("warnings_max_diffs")
if "runas_username" in kwargs:
self.runas_username = kwargs.pop("runas_username")
if "transaction_comment" in kwargs:
self.transaction_comment = kwargs.pop("transaction_comment")
return kwargs
def begin_transaction(self):
@ -946,11 +982,41 @@ class ToWuttaHandler(ToSqlalchemyHandler):
def make_target_session(self):
"""
Call
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`
and return the result.
This creates a typical :term:`db session` for the app by
calling
:meth:`~wuttjamaican:wuttjamaican.app.AppHandler.make_session()`.
It then may "customize" the session slightly. These
customizations only are relevant if Wutta-Continuum versioning
is enabled:
If :attr:`~ImportHandler.runas_username` is set, the
responsible user (``continuum_user_id``) will be set for the
new session as well.
Similarly, if :attr:`~ImportHandler.transaction_comment` is
set, it (``continuum_comment``) will also be set for the new
session.
:returns: :class:`~wuttjamaican:wuttjamaican.db.sess.Session`
instance.
"""
return self.app.make_session()
model = self.app.model
session = self.app.make_session()
# set runas user in case continuum versioning is enabled
if self.runas_username:
if user := (
session.query(model.User)
.filter_by(username=self.runas_username)
.first()
):
session.info["continuum_user_id"] = user.uuid
else:
log.warning("runas username not found: %s", self.runas_username)
# set comment in case continuum versioning is enabled
if self.transaction_comment:
session.info["continuum_comment"] = self.transaction_comment
return session

View file

@ -92,13 +92,6 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
See also :attr:`continuum_uow`.
"""
continuum_comment = None
def consume_kwargs(self, kwargs):
kwargs = super().consume_kwargs(kwargs)
self.continuum_comment = kwargs.pop("comment", None)
return kwargs
def begin_target_transaction(self):
# pylint: disable=line-too-long
"""
@ -128,8 +121,8 @@ class FromWuttaToVersions(FromWuttaHandler, ToWuttaHandler):
self.continuum_txn = self.continuum_uow.create_transaction(self.target_session)
if self.continuum_comment:
self.continuum_txn.meta = {"comment": self.continuum_comment}
if self.transaction_comment:
self.continuum_txn.meta = {"comment": self.transaction_comment}
def get_importer_kwargs(self, key, **kwargs):
"""

View file

@ -1,8 +1,9 @@
# -*- coding: utf-8; -*-
import inspect
import sys
from unittest import TestCase
from unittest.mock import patch
from unittest.mock import patch, Mock
from wuttasync.cli import base as mod
from wuttjamaican.testing import DataTestCase
@ -44,12 +45,97 @@ class TestImportCommandHandler(DataTestCase):
)
with patch.object(handler, "list_models") as list_models:
handler.run({"list_models": True})
list_models.assert_called_once_with({"list_models": True})
ctx = Mock(params={"list_models": True})
handler.run(ctx)
list_models.assert_called_once_with(ctx.params)
class Object:
def __init__(self, **kw):
self.__dict__.update(kw)
with patch.object(handler, "import_handler") as import_handler:
handler.run({"models": []})
import_handler.process_data.assert_called_once_with()
parent = Mock(
params={
"runas_username": "fred",
"comment": "hello world",
}
)
# TODO: why can't we just use Mock here? the parent attr is problematic
ctx = Object(params={"models": []}, parent=parent)
handler.run(ctx)
import_handler.process_data.assert_called_once_with(
runas_username="fred",
transaction_comment="hello world",
)
def test_run_missing_input(self):
handler = self.make_handler(
import_handler="wuttasync.importing.csv:FromCsvToWutta"
)
class Object:
def __init__(self, **kw):
self.__dict__.update(kw)
# fails without input_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={},
parent=Object(params={}),
)
try:
handler.run(ctx)
except RuntimeError:
pass
exit_.assert_called_once_with(1)
# runs with input_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={"input_file_path": self.tempdir},
parent=Object(
params={},
),
)
self.assertRaises(FileNotFoundError, handler.run, ctx)
exit_.assert_not_called()
def test_run_missing_output(self):
handler = self.make_handler(
import_handler="wuttasync.exporting.csv:FromWuttaToCsv"
)
class Object:
def __init__(self, **kw):
self.__dict__.update(kw)
# fails without output_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={},
parent=Object(params={}),
)
try:
handler.run(ctx)
except RuntimeError:
pass
exit_.assert_called_once_with(1)
# runs with output_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={"output_file_path": self.tempdir},
parent=Object(
params={},
),
)
# self.assertRaises(FileNotFoundError, handler.run, ctx)
handler.run(ctx)
exit_.assert_not_called()
def test_list_models(self):
handler = self.make_handler(
@ -80,6 +166,23 @@ class TestImporterCommand(TestCase):
self.assertIn("dry_run", sig2.parameters)
class TestFileExporterCommand(TestCase):
def test_basic(self):
def myfunc(ctx, **kwargs):
pass
sig1 = inspect.signature(myfunc)
self.assertIn("kwargs", sig1.parameters)
self.assertNotIn("dry_run", sig1.parameters)
self.assertNotIn("output_file_path", sig1.parameters)
wrapt = mod.file_export_command(myfunc)
sig2 = inspect.signature(wrapt)
self.assertNotIn("kwargs", sig2.parameters)
self.assertIn("dry_run", sig2.parameters)
self.assertIn("output_file_path", sig2.parameters)
class TestFileImporterCommand(TestCase):
def test_basic(self):

View file

@ -0,0 +1,22 @@
# -*- coding: utf-8; -*-
from unittest import TestCase
from unittest.mock import MagicMock, patch
from wuttasync.cli import export_csv as mod, ImportCommandHandler
class TestExportCsv(TestCase):
def test_basic(self):
params = {
"models": [],
"create": True,
"update": True,
"delete": False,
"dry_run": True,
}
ctx = MagicMock(params=params)
with patch.object(ImportCommandHandler, "run") as run:
mod.export_csv(ctx)
run.assert_called_once_with(ctx)

View file

@ -19,4 +19,4 @@ class TestImportCsv(TestCase):
ctx = MagicMock(params=params)
with patch.object(ImportCommandHandler, "run") as run:
mod.import_csv(ctx)
run.assert_called_once_with(params)
run.assert_called_once_with(ctx)

View file

@ -19,4 +19,4 @@ class TestImportCsv(TestCase):
ctx = MagicMock(params=params)
with patch.object(ImportCommandHandler, "run") as run:
mod.import_versions(ctx)
run.assert_called_once_with(params)
run.assert_called_once_with(ctx)

View file

@ -0,0 +1,99 @@
# -*- coding: utf-8; -*-
from unittest.mock import patch
from wuttjamaican.testing import DataTestCase
from wuttasync.exporting import base as mod, ExportHandler
class TestToFile(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ExportHandler(self.config)
def make_exporter(self, **kwargs):
kwargs.setdefault("handler", self.handler)
return mod.ToFile(self.config, **kwargs)
def test_setup(self):
model = self.app.model
# output file is opened
exp = self.make_exporter(model_class=model.Setting)
self.assertFalse(exp.dry_run)
with patch.object(exp, "open_output_file") as open_output_file:
exp.setup()
open_output_file.assert_called_once_with()
# but not if in dry run mode
with patch.object(self.handler, "dry_run", new=True):
exp = self.make_exporter(model_class=model.Setting)
self.assertTrue(exp.dry_run)
with patch.object(exp, "open_output_file") as open_output_file:
exp.setup()
open_output_file.assert_not_called()
def test_teardown(self):
model = self.app.model
# output file is closed
exp = self.make_exporter(model_class=model.Setting)
self.assertFalse(exp.dry_run)
with patch.object(exp, "close_output_file") as close_output_file:
exp.teardown()
close_output_file.assert_called_once_with()
# but not if in dry run mode
with patch.object(self.handler, "dry_run", new=True):
exp = self.make_exporter(model_class=model.Setting)
self.assertTrue(exp.dry_run)
with patch.object(exp, "close_output_file") as close_output_file:
exp.teardown()
close_output_file.assert_not_called()
def test_get_output_file_path(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
# output path must be set
self.assertRaises(ValueError, exp.get_output_file_path)
# path is guessed from dir+filename
path1 = self.write_file("data1.txt", "")
exp.output_file_path = self.tempdir
exp.output_file_name = "data1.txt"
self.assertEqual(exp.get_output_file_path(), path1)
# path can be explicitly set
path2 = self.write_file("data2.txt", "")
exp.output_file_path = path2
self.assertEqual(exp.get_output_file_path(), path2)
def test_get_output_file_name(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
# name cannot be guessed
self.assertRaises(NotImplementedError, exp.get_output_file_name)
# name can be explicitly set
exp.output_file_name = "data.txt"
self.assertEqual(exp.get_output_file_name(), "data.txt")
def test_open_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
self.assertRaises(NotImplementedError, exp.open_output_file)
def test_close_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
path = self.write_file("data.txt", "")
with open(path, "wt") as f:
exp.output_file = f
with patch.object(f, "close") as close:
exp.close_output_file()
close.assert_called_once_with()

209
tests/exporting/test_csv.py Normal file
View file

@ -0,0 +1,209 @@
# -*- coding: utf-8; -*-
import csv
import io
from unittest.mock import patch
from wuttjamaican.testing import DataTestCase
from wuttasync.exporting import csv as mod, ExportHandler
from wuttasync.importing import FromWuttaHandler, FromWutta
class TestToCsv(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ExportHandler(self.config)
def make_exporter(self, **kwargs):
kwargs.setdefault("handler", self.handler)
kwargs.setdefault("output_file_path", self.tempdir)
return mod.ToCsv(self.config, **kwargs)
def test_get_output_file_name(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
# name can be guessed
self.assertEqual(exp.get_output_file_name(), "Setting.csv")
# name can be explicitly set
exp.output_file_name = "data.txt"
self.assertEqual(exp.get_output_file_name(), "data.txt")
def test_open_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
self.assertIsNone(exp.output_file)
self.assertIsNone(exp.output_writer)
exp.open_output_file()
try:
self.assertIsInstance(exp.output_file, io.TextIOBase)
self.assertIsInstance(exp.output_writer, csv.DictWriter)
finally:
exp.output_file.close()
def test_close_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
self.assertIsNone(exp.output_file)
self.assertIsNone(exp.output_writer)
exp.open_output_file()
self.assertIsNotNone(exp.output_file)
self.assertIsNotNone(exp.output_writer)
exp.close_output_file()
self.assertIsNone(exp.output_file)
self.assertIsNone(exp.output_writer)
def test_coerce_csv(self):
model = self.app.model
# string value
exp = self.make_exporter(model_class=model.Setting)
result = exp.coerce_csv({"name": "foo", "value": "bar"})
self.assertEqual(result, {"name": "foo", "value": "bar"})
# null value converts to empty string
result = exp.coerce_csv({"name": "foo", "value": None})
self.assertEqual(result, {"name": "foo", "value": ""})
# float value passed thru as-is
result = exp.coerce_csv({"name": "foo", "value": 12.34})
self.assertEqual(result, {"name": "foo", "value": 12.34})
self.assertIsInstance(result["value"], float)
def test_update_target_object(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
exp.setup()
with patch.object(exp, "output_writer") as output_writer:
# writer is called for normal run
data = {"name": "foo", "value": "bar"}
exp.update_target_object(None, data)
output_writer.writerow.assert_called_once_with(data)
# but not called for dry run
output_writer.writerow.reset_mock()
with patch.object(self.handler, "dry_run", new=True):
exp.update_target_object(None, data)
output_writer.writerow.assert_not_called()
exp.teardown()
class MockMixinExporter(mod.FromSqlalchemyToCsvMixin, FromWutta, mod.ToCsv):
pass
class TestFromSqlalchemyToCsvMixin(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ExportHandler(self.config)
def make_exporter(self, **kwargs):
kwargs.setdefault("handler", self.handler)
return MockMixinExporter(self.config, **kwargs)
def test_model_title(self):
model = self.app.model
exp = self.make_exporter(source_model_class=model.Setting)
# default comes from model class
self.assertEqual(exp.get_model_title(), "Setting")
# but can override
exp.model_title = "Widget"
self.assertEqual(exp.get_model_title(), "Widget")
def test_get_simple_fields(self):
model = self.app.model
exp = self.make_exporter(source_model_class=model.Setting)
# default comes from model class
self.assertEqual(exp.get_simple_fields(), ["name", "value"])
# but can override
exp.simple_fields = ["name"]
self.assertEqual(exp.get_simple_fields(), ["name"])
# no default if no model class
exp = self.make_exporter()
self.assertEqual(exp.get_simple_fields(), [])
def test_normalize_source_object(self):
model = self.app.model
exp = self.make_exporter(source_model_class=model.Setting)
setting = model.Setting(name="foo", value="bar")
data = exp.normalize_source_object(setting)
self.assertEqual(data, {"name": "foo", "value": "bar"})
def test_make_object(self):
model = self.app.model
# normal
exp = self.make_exporter(source_model_class=model.Setting)
obj = exp.make_object()
self.assertIsInstance(obj, model.Setting)
# no model_class
exp = self.make_exporter()
self.assertRaises(TypeError, exp.make_object)
class MockMixinHandler(
mod.FromSqlalchemyToCsvHandlerMixin, FromWuttaHandler, mod.ToCsvHandler
):
FromImporterBase = FromWutta
class TestFromSqlalchemyToCsvHandlerMixin(DataTestCase):
def make_handler(self, **kwargs):
return MockMixinHandler(self.config, **kwargs)
def test_get_source_model(self):
with patch.object(
mod.FromSqlalchemyToCsvHandlerMixin, "define_importers", return_value={}
):
handler = self.make_handler()
self.assertRaises(NotImplementedError, handler.get_source_model)
def test_define_importers(self):
model = self.app.model
with patch.object(
mod.FromSqlalchemyToCsvHandlerMixin, "get_source_model", return_value=model
):
handler = self.make_handler()
importers = handler.define_importers()
self.assertIn("Setting", importers)
self.assertTrue(issubclass(importers["Setting"], FromWutta))
self.assertTrue(issubclass(importers["Setting"], mod.ToCsv))
self.assertIn("User", importers)
self.assertIn("Person", importers)
self.assertIn("Role", importers)
def test_make_importer_factory(self):
model = self.app.model
with patch.object(
mod.FromSqlalchemyToCsvHandlerMixin, "define_importers", return_value={}
):
handler = self.make_handler()
factory = handler.make_importer_factory(model.Setting, "Setting")
self.assertTrue(issubclass(factory, FromWutta))
self.assertTrue(issubclass(factory, mod.ToCsv))
class TestFromWuttaToCsv(DataTestCase):
def make_handler(self, **kwargs):
return mod.FromWuttaToCsv(self.config, **kwargs)
def test_get_source_model(self):
handler = self.make_handler()
self.assertIs(handler.get_source_model(), self.app.model)

View file

@ -0,0 +1,4 @@
# -*- coding: utf-8; -*-
# nothing to test yet really, just ensuring coverage
from wuttasync.exporting import handlers as mod

View file

@ -271,6 +271,9 @@ class Example(Base):
flag = sa.Column(sa.Boolean(), nullable=False)
optional_flag = sa.Column(sa.Boolean(), nullable=True)
date = sa.Column(sa.Date(), nullable=False)
optional_date = sa.Column(sa.Date(), nullable=True)
dt = sa.Column(sa.DateTime(), nullable=False)
optional_dt = sa.Column(sa.DateTime(), nullable=True)
@ -285,7 +288,7 @@ class TestMakeCoercers(TestCase):
def test_basic(self):
coercers = mod.make_coercers(Example)
self.assertEqual(len(coercers), 12)
self.assertEqual(len(coercers), 14)
self.assertIs(coercers["id"], mod.coerce_integer)
self.assertIs(coercers["optional_id"], mod.coerce_integer)
@ -293,6 +296,8 @@ class TestMakeCoercers(TestCase):
self.assertIs(coercers["optional_name"], mod.coerce_string_nullable)
self.assertIs(coercers["flag"], mod.coerce_boolean)
self.assertIs(coercers["optional_flag"], mod.coerce_boolean_nullable)
self.assertIs(coercers["date"], mod.coerce_date)
self.assertIs(coercers["optional_date"], mod.coerce_date)
self.assertIs(coercers["dt"], mod.coerce_datetime)
self.assertIs(coercers["optional_dt"], mod.coerce_datetime)
self.assertIs(coercers["dec"], mod.coerce_decimal)
@ -322,6 +327,12 @@ class TestMakeCoercer(TestCase):
func = mod.make_coercer(Example.optional_flag)
self.assertIs(func, mod.coerce_boolean_nullable)
func = mod.make_coercer(Example.date)
self.assertIs(func, mod.coerce_date)
func = mod.make_coercer(Example.optional_date)
self.assertIs(func, mod.coerce_date)
func = mod.make_coercer(Example.dt)
self.assertIs(func, mod.coerce_datetime)
@ -365,6 +376,15 @@ class TestCoercers(TestCase):
self.assertIsNone(mod.coerce_boolean_nullable(""))
def test_coerce_date(self):
self.assertIsNone(mod.coerce_date(""))
value = mod.coerce_date("2025-10-19")
self.assertIsInstance(value, datetime.date)
self.assertEqual(value, datetime.date(2025, 10, 19))
self.assertRaises(ValueError, mod.coerce_date, "XXX")
def test_coerce_datetime(self):
self.assertIsNone(mod.coerce_datetime(""))

View file

@ -27,6 +27,13 @@ class TestImportHandler(DataTestCase):
handler.target_title = "Wutta"
self.assertEqual(str(handler), "CSV → Wutta")
def test_actioner(self):
handler = self.make_handler()
self.assertEqual(handler.actioner, "importer")
handler.orientation = mod.Orientation.EXPORT
self.assertEqual(handler.actioner, "exporter")
def test_actioning(self):
handler = self.make_handler()
self.assertEqual(handler.actioning, "importing")
@ -190,6 +197,22 @@ class TestImportHandler(DataTestCase):
self.assertNotIn("warnings_max_diffs", kw)
self.assertEqual(handler.warnings_max_diffs, 30)
# runas_username (consumed)
self.assertIsNone(handler.runas_username)
kw["runas_username"] = "fred"
result = handler.consume_kwargs(kw)
self.assertIs(result, kw)
self.assertNotIn("runas_username", kw)
self.assertEqual(handler.runas_username, "fred")
# transaction_comment (consumed)
self.assertIsNone(handler.transaction_comment)
kw["transaction_comment"] = "hello world"
result = handler.consume_kwargs(kw)
self.assertIs(result, kw)
self.assertNotIn("transaction_comment", kw)
self.assertEqual(handler.transaction_comment, "hello world")
def test_define_importers(self):
handler = self.make_handler()
importers = handler.define_importers()
@ -490,11 +513,41 @@ class TestToWuttaHandler(DataTestCase):
self.assertEqual(handler.get_target_title(), "what_about_this")
def test_make_target_session(self):
model = self.app.model
handler = self.make_handler()
# makes "new" (mocked in our case) app session
fred = model.User(username="fred")
self.session.add(fred)
self.session.commit()
# makes "new" (mocked in our case) app session, with no runas
# username set by default
with patch.object(self.app, "make_session") as make_session:
make_session.return_value = self.session
session = handler.make_target_session()
make_session.assert_called_once_with()
self.assertIs(session, self.session)
self.assertNotIn("continuum_user_id", session.info)
self.assertNotIn("continuum_user_id", self.session.info)
# runas user also should not be set, if username is not valid
handler.runas_username = "freddie"
with patch.object(self.app, "make_session") as make_session:
make_session.return_value = self.session
session = handler.make_target_session()
make_session.assert_called_once_with()
self.assertIs(session, self.session)
self.assertNotIn("continuum_user_id", session.info)
self.assertNotIn("continuum_user_id", self.session.info)
# this time we should have runas user properly set
handler.runas_username = "fred"
with patch.object(self.app, "make_session") as make_session:
make_session.return_value = self.session
session = handler.make_target_session()
make_session.assert_called_once_with()
self.assertIs(session, self.session)
self.assertIn("continuum_user_id", session.info)
self.assertEqual(session.info["continuum_user_id"], fred.uuid)
self.assertIn("continuum_user_id", self.session.info)
self.assertEqual(self.session.info["continuum_user_id"], fred.uuid)

View file

@ -14,20 +14,6 @@ class TestFromWuttaToVersions(VersionTestCase):
def make_handler(self, **kwargs):
return mod.FromWuttaToVersions(self.config, **kwargs)
def test_consume_kwargs(self):
# no comment by default
handler = self.make_handler()
kw = handler.consume_kwargs({})
self.assertEqual(kw, {})
self.assertIsNone(handler.continuum_comment)
# but can provide one
handler = self.make_handler()
kw = handler.consume_kwargs({"comment": "yeehaw"})
self.assertEqual(kw, {})
self.assertEqual(handler.continuum_comment, "yeehaw")
def test_begin_target_transaction(self):
model = self.app.model
txncls = continuum.transaction_class(model.User)
@ -44,7 +30,7 @@ class TestFromWuttaToVersions(VersionTestCase):
# with comment
handler = self.make_handler()
handler.continuum_comment = "yeehaw"
handler.transaction_comment = "yeehaw"
handler.begin_target_transaction()
self.assertIn("comment", handler.continuum_txn.meta)
self.assertEqual(handler.continuum_txn.meta["comment"], "yeehaw")

View file

@ -1,5 +1,7 @@
# -*- coding: utf-8; -*-
from unittest.mock import patch
from wuttjamaican.testing import ConfigTestCase
from wuttasync import app as mod
@ -16,6 +18,16 @@ class FromCsvToPoser(FromCsvToWutta):
pass
class FromFooToBaz1(ImportHandler):
source_key = "foo"
target_key = "baz"
class FromFooToBaz2(ImportHandler):
source_key = "foo"
target_key = "baz"
class TestWuttaSyncAppProvider(ConfigTestCase):
def test_get_all_import_handlers(self):
@ -100,6 +112,18 @@ class TestWuttaSyncAppProvider(ConfigTestCase):
any([h.get_key() == "import.to_versions.from_wutta" for h in handlers])
)
# nothing returned if multiple handlers found but none are designated
with patch.object(
self.app.providers["wuttasync"],
"get_all_import_handlers",
return_value=[FromFooToBaz1, FromFooToBaz2],
):
handlers = self.app.get_designated_import_handlers()
baz_handlers = [
h for h in handlers if h.get_key() == "import.to_baz.from_foo"
]
self.assertEqual(len(baz_handlers), 0)
def test_get_import_handler(self):
# make sure a basic fetch works