feat: add support for wutta export-csv command

This commit is contained in:
Lance Edgar 2026-01-03 15:41:47 -06:00
parent c873cc462e
commit 61deaad251
21 changed files with 1186 additions and 16 deletions

View file

@ -0,0 +1,6 @@
``wuttasync.exporting.base``
============================
.. automodule:: wuttasync.exporting.base
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting.csv``
===========================
.. automodule:: wuttasync.exporting.csv
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting.handlers``
================================
.. automodule:: wuttasync.exporting.handlers
:members:

View file

@ -0,0 +1,6 @@
``wuttasync.exporting``
=======================
.. automodule:: wuttasync.exporting
:members:

View file

@ -76,6 +76,10 @@ cf. :doc:`rattail-manual:data/sync/index`.
api/wuttasync.cli.import_csv
api/wuttasync.cli.import_versions
api/wuttasync.emails
api/wuttasync.exporting
api/wuttasync.exporting.base
api/wuttasync.exporting.csv
api/wuttasync.exporting.handlers
api/wuttasync.importing
api/wuttasync.importing.base
api/wuttasync.importing.csv

View file

@ -28,6 +28,7 @@ requires-python = ">= 3.8"
dependencies = [
"humanize",
"makefun",
"rich",
"SQLAlchemy-Utils",
"WuttJamaican[db]>=0.28.1",
]
@ -42,6 +43,7 @@ tests = ["pylint", "pytest", "pytest-cov", "tox", "Wutta-Continuum>=0.3.0"]
wuttasync = "wuttasync.app:WuttaSyncAppProvider"
[project.entry-points."wuttasync.importing"]
"export.to_csv.from_wutta" = "wuttasync.exporting.csv:FromWuttaToCsv"
"import.to_versions.from_wutta" = "wuttasync.importing.versions:FromWuttaToVersions"
"import.to_wutta.from_csv" = "wuttasync.importing.csv:FromCsvToWutta"

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -26,12 +26,19 @@ WuttaSync - ``wutta`` subcommands
This namespace exposes the following:
* :func:`~wuttasync.cli.base.import_command()`
* :func:`~wuttasync.cli.base.file_export_command()`
* :func:`~wuttasync.cli.base.file_import_command()`
* :class:`~wuttasync.cli.base.ImportCommandHandler`
"""
from .base import import_command, file_import_command, ImportCommandHandler
from .base import (
import_command,
file_export_command,
file_import_command,
ImportCommandHandler,
)
# nb. must bring in all modules for discovery to work
from . import export_csv
from . import import_csv
from . import import_versions

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -32,10 +32,12 @@ from typing import List, Optional
from typing_extensions import Annotated
import makefun
import rich
import typer
from wuttjamaican.app import GenericHandler
from wuttasync.importing import ImportHandler
from wuttasync.importing import ImportHandler, FromFileHandler
from wuttasync.exporting import ToFileHandler
log = logging.getLogger(__name__)
@ -123,9 +125,30 @@ class ImportCommandHandler(GenericHandler):
self.list_models(ctx.params)
return
# otherwise we'll process some data
# otherwise we'll (hopefully) process some data
log.debug("using handler: %s", self.import_handler.get_spec())
# but first, some extra checks for certain file-based
# handlers. this must be done here, because these CLI params
# are not technically required (otherwise typer would handle
# this instead of us here). and that is because we want to
# allow user to specify --list without needing to also specify
# --input or --output
if isinstance(self.import_handler, FromFileHandler):
if not ctx.params.get("input_file_path"):
rich.print(
"\n[bold yellow]must specify --input folder/file path[/bold yellow]\n",
file=sys.stderr,
)
sys.exit(1)
elif isinstance(self.import_handler, ToFileHandler):
if not ctx.params.get("output_file_path"):
rich.print(
"\n[bold yellow]must specify --output folder/file path[/bold yellow]\n",
file=sys.stderr,
)
sys.exit(1)
# all params from caller will be passed along
kw = dict(ctx.params)
@ -136,7 +159,7 @@ class ImportCommandHandler(GenericHandler):
kw["transaction_comment"] = comment
# sort out which models to process
models = kw.pop("models")
models = kw.pop("models", None)
if not models:
models = list(self.import_handler.importers)
log.debug(
@ -170,7 +193,7 @@ def import_command_template( # pylint: disable=unused-argument,too-many-argumen
models: Annotated[
Optional[List[str]],
typer.Argument(
help="Model(s) to process. Can specify one or more, "
help="Target model(s) to process. Specify one or more, "
"or omit to process default models."
),
] = None,
@ -324,17 +347,65 @@ def import_command(fn):
return makefun.create_function(final_sig, fn)
def file_import_command_template( # pylint: disable=unused-argument
input_file_path: Annotated[
def file_export_command_template( # pylint: disable=unused-argument
# nb. technically this is required, but not if doing --list
# (so we cannot mark it required here, for that reason)
output_file_path: Annotated[
Path,
typer.Option(
"--input-path",
"--output",
"-o",
exists=True,
file_okay=True,
dir_okay=True,
help="Path to input file(s). Can be a folder "
"if app logic can guess the filename(s); "
"otherwise must be complete file path.",
help="Path to output folder. Or full path to output file "
"if only running one target model.",
),
] = None,
):
"""
Stub function to provide signature for exporter commands which
produce data file(s) as output. Used with
:func:`file_export_command`.
"""
def file_export_command(fn):
"""
Decorator for file export commands. Adds common params based on
:func:`file_export_command_template`.
"""
original_sig = inspect.signature(fn)
plain_import_sig = inspect.signature(import_command_template)
file_export_sig = inspect.signature(file_export_command_template)
desired_params = list(plain_import_sig.parameters.values()) + list(
file_export_sig.parameters.values()
)
params = list(original_sig.parameters.values())
for i, param in enumerate(desired_params):
params.insert(i + 1, param)
# remove the **kwargs param
params.pop(-1)
final_sig = original_sig.replace(parameters=params)
return makefun.create_function(final_sig, fn)
def file_import_command_template( # pylint: disable=unused-argument
# nb. technically this is required, but not if doing --list
# (so we cannot mark it required here, for that reason)
input_file_path: Annotated[
Path,
typer.Option(
"--input",
"-i",
exists=True,
file_okay=True,
dir_okay=True,
help="Path to input folder. Or full path to input file "
"if only running one target model.",
),
] = None,
):

View file

@ -0,0 +1,42 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
See also: :ref:`wutta-export-csv`
"""
import typer
from wuttjamaican.cli import wutta_typer
from .base import file_export_command, ImportCommandHandler
@wutta_typer.command()
@file_export_command
def export_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
"""
Export data from Wutta DB to CSV file(s)
"""
config = ctx.parent.wutta_config
handler = ImportCommandHandler(config, key="export.to_csv.from_wutta")
handler.run(ctx)

View file

@ -0,0 +1,43 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Data Import / Export Framework
This namespace exposes the following:
* :enum:`~wuttasync.importing.handlers.Orientation`
And some :term:`export handler <import handler>` base classes:
* :class:`~wuttasync.exporting.handlers.ExportHandler`
* :class:`~wuttasync.exporting.handlers.ToFileHandler`
And some :term:`exporter <importer>` base classes:
* :class:`~wuttasync.exporting.base.ToFile`
See also the :mod:`wuttasync.importing` module.
"""
from .handlers import Orientation, ExportHandler, ToFileHandler
from .base import ToFile

View file

@ -0,0 +1,166 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Data exporter base classes
"""
import os
from wuttasync.importing import Importer
class ToFile(Importer):
"""
Base class for importer/exporter using output file as data target.
Depending on the subclass, it may be able to "guess" (at least
partially) the path to the output file. If not, and/or to avoid
ambiguity, the caller must specify the file path.
In most cases caller may specify any of these via kwarg to the
class constructor, or e.g.
:meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`:
* :attr:`output_file_path`
* :attr:`output_file_name`
The subclass itself can also specify via override of these
methods:
* :meth:`get_output_file_path()`
* :meth:`get_output_file_name()`
And of course subclass must override these too:
* :meth:`open_output_file()`
* :meth:`close_output_file()`
* (and see also :attr:`output_file`)
"""
output_file_path = None
"""
Path to output folder, or file.
The ideal usage is to set this to the output *folder* path. That
allows the handler to run several importers in one go. The same
output folder path is given to each importer; they then each
determine their own output filename within that.
But you can also set this to the full output folder + file path,
e.g. if you're just running one importer. This would override
the importer's own logic for determining output filename.
See also :meth:`get_output_file_path()` and
:meth:`get_output_file_name()`.
"""
output_file_name = None
"""
Optional static output file name (sans folder path).
If set, this will be used as output filename instead of the
importer determining one on its own.
See also :meth:`get_output_file_name()`.
"""
output_file = None
"""
Handle to the open output file, if applicable. May be set by
:meth:`open_output_file()` for later reference within
:meth:`close_output_file()`.
"""
def setup(self):
"""
Open the output file. See also :meth:`open_output_file()`.
"""
if not self.dry_run:
self.open_output_file()
def teardown(self):
"""
Close the output file. See also :meth:`close_output_file()`.
"""
if not self.dry_run:
self.close_output_file()
def get_output_file_path(self):
"""
This must return the full path to output file.
Default logic inspects :attr:`output_file_path`; if that
points to a folder then it is combined with
:meth:`get_output_file_name()`. Otherwise it's returned
as-is.
:returns: Path to output file, as string
"""
path = self.output_file_path
if not path:
raise ValueError("must set output_file_path")
if os.path.isdir(path):
filename = self.get_output_file_name()
return os.path.join(path, filename)
return path
def get_output_file_name(self):
"""
This must return the output filename, sans folder path.
Default logic will return :attr:`output_file_name` if set,
otherwise raise error.
:returns: Output filename, sans folder path
"""
if self.output_file_name:
return self.output_file_name
raise NotImplementedError("can't guess output filename")
def open_output_file(self):
"""
Open the output file for writing target data.
Subclass must override to specify how this happens; default
logic is not implemented. Remember to set :attr:`output_file`
if applicable for reference when closing.
See also :attr:`get_output_file_path()` and
:meth:`close_output_file()`.
"""
raise NotImplementedError
def close_output_file(self):
"""
Close the output file for target data.
Subclass must override to specify how this happens; default
logic blindly calls the ``close()`` method on whatever
:attr:`output_file` happens to point to.
See also :attr:`open_output_file()`.
"""
self.output_file.close()

View file

@ -0,0 +1,321 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Exporting to CSV
"""
import csv
import logging
from collections import OrderedDict
import sqlalchemy as sa
from sqlalchemy_utils.functions import get_primary_keys, get_columns
from wuttjamaican.db.util import make_topo_sortkey
from wuttasync.importing import FromWuttaHandler, FromWutta
from wuttasync.exporting import ToFileHandler, ToFile
log = logging.getLogger(__name__)
class ToCsv(ToFile): # pylint: disable=abstract-method
"""
Base class for exporter using CSV file as data target.
This inherits from :class:`~wuttasync.exporting.base.ToFile`.
"""
output_writer = None
"""
While the output file is open, this will reference a
:class:`python:csv.DictWriter` instance.
"""
csv_encoding = "utf_8"
"""
Encoding used for the CSV output file.
You can specify an override if needed when calling
:meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`.
"""
def get_output_file_name(self): # pylint: disable=empty-docstring
""" """
if self.output_file_name:
return self.output_file_name
model_title = self.get_model_title()
return f"{model_title}.csv"
def open_output_file(self):
"""
Opens the output CSV file for writing.
This calls
:meth:`~wuttasync.exporting.base.ToFile.get_output_file_path()`
and opens that file. It sets
:attr:`~wuttasync.exporting.base.ToFile.output_file` and also
:attr:`output_writer`. And it calls
:meth:`write_output_header()` to write the field header row.
"""
path = self.get_output_file_path()
log.debug("opening output file: %s", path)
self.output_file = open( # pylint: disable=consider-using-with
path, "wt", encoding=self.csv_encoding
)
self.output_writer = csv.DictWriter(
self.output_file,
self.fields,
# quoting=csv.QUOTE_NONNUMERIC
)
self.write_output_header()
def write_output_header(self):
"""
Write the field header row to the CSV file.
Default logic calls
:meth:`~python:csv.DictWriter.writeheader()` on the
:attr:`output_writer` instance.
"""
self.output_writer.writeheader()
def close_output_file(self): # pylint: disable=empty-docstring
""" """
self.output_writer = None
self.output_file.close()
self.output_file = None
def update_target_object(self, obj, source_data, target_data=None):
"""
In a CSV export the assumption is we always start with an
empty file, so "create" is the only logical action for each
record - there are no updates or deletes per se.
But under the hood, this method is used for create as well, so
we override it and actually write the record to CSV file.
Unless :attr:`~wuttasync.importing.base.Importer.dry_run` is
true, this calls :meth:`~python:csv.csvwriter.writerow()` on
the :attr:`output_writer` instance.
See also parent method docs,
:meth:`~wuttasync.importing.base.Importer.update_target_object()`
"""
data = self.coerce_csv(source_data)
if not self.dry_run:
self.output_writer.writerow(data)
return data
def coerce_csv(self, data): # pylint: disable=missing-function-docstring
coerced = {}
for field in self.fields:
value = data[field]
if value is None:
value = ""
elif isinstance(value, (int, float)):
pass
else:
value = str(value)
coerced[field] = value
return coerced
class FromSqlalchemyToCsvMixin:
"""
Mixin class for SQLAlchemy ORM CSV :term:`exporters <importer>`.
Such exporters are generated automatically by
:class:`FromSqlalchemyToCsvHandlerMixin`, so you won't typically
reference this mixin class directly.
This mixin effectively behaves like the
:attr:`~wuttasync.importing.base.Importer.model_class` represents
the source side instead of the target. It uses
:attr:`~wuttasync.importing.base.FromSqlalchemy.source_model_class`
instead, for automatic things like inspecting the fields list.
"""
def get_model_title(self): # pylint: disable=missing-function-docstring
if hasattr(self, "model_title"):
return self.model_title
return self.source_model_class.__name__
def get_simple_fields(self): # pylint: disable=missing-function-docstring
if hasattr(self, "simple_fields"):
return self.simple_fields
try:
fields = get_columns(self.source_model_class)
except sa.exc.NoInspectionAvailable:
return []
return list(fields.keys())
def normalize_source_object(
self, obj
): # pylint: disable=missing-function-docstring
fields = self.get_fields()
fields = [f for f in self.get_simple_fields() if f in fields]
data = {field: getattr(obj, field) for field in fields}
return data
def make_object(self): # pylint: disable=missing-function-docstring
return self.source_model_class()
class FromSqlalchemyToCsvHandlerMixin:
"""
Mixin class for SQLAlchemy ORM CSV :term:`export handlers
<import handler>`.
This knows how to dynamically generate :term:`exporter <importer>`
classes to represent the models in the source ORM. Such classes
will inherit from :class:`FromSqlalchemyToCsvMixin`, in addition
to whatever :attr:`FromImporterBase` and :attr:`ToImporterBase`
reference.
That all happens within :meth:`define_importers()`.
"""
target_key = "csv"
generic_target_title = "CSV"
# nb. subclass must define this
FromImporterBase = None
"""
For a handler to use this mixin, it must set this to a valid base
class for the ORM source side. The :meth:`define_importers()`
logic will use this when dynamically generating new exporter
classes.
"""
ToImporterBase = ToCsv
"""
This must be set to a valid base class for the CSV target side.
Default is :class:`ToCsv` which should typically be fine; you can
change if needed.
"""
def get_source_model(self):
"""
This should return the :term:`app model` or a similar module
containing data model classes for the source side.
The source model is used to dynamically generate a set of
exporters (e.g. one per table in the source DB) which can use
CSV file as data target. See also :meth:`define_importers()`.
Subclass must override this if needed; default behavior is not
implemented.
"""
raise NotImplementedError
def define_importers(self):
"""
This mixin overrides typical (manual) importer definition, and
instead dynamically generates a set of exporters, e.g. one per
table in the source DB.
It does this based on the source model, as returned by
:meth:`get_source_model()`. It calls
:meth:`make_importer_factory()` for each model class found.
"""
importers = {}
model = self.get_source_model()
# mostly try to make an importer for every data model
for name in dir(model):
cls = getattr(model, name)
if (
isinstance(cls, type)
and issubclass(cls, model.Base)
and cls is not model.Base
):
importers[name] = self.make_importer_factory(cls, name)
# sort importers according to schema topography
topo_sortkey = make_topo_sortkey(model)
importers = OrderedDict(
[(name, importers[name]) for name in sorted(importers, key=topo_sortkey)]
)
return importers
def make_importer_factory(self, model_class, name):
"""
Generate a new :term:`exporter <importer>` class, targeting
the given :term:`data model` class.
The newly-created class will inherit from:
* :class:`FromSqlalchemyToCsvMixin`
* :attr:`FromImporterBase`
* :attr:`ToImporterBase`
:param model_class: A data model class.
:param name: The "model name" for the importer/exporter. New
class name will be based on this, so e.g. ``Widget`` model
name becomes ``WidgetImporter`` class name.
:returns: The new class, meant to process import/export
targeting the given data model.
"""
return type(
f"{name}Importer",
(FromSqlalchemyToCsvMixin, self.FromImporterBase, self.ToImporterBase),
{
"source_model_class": model_class,
"default_keys": list(get_primary_keys(model_class)),
},
)
class ToCsvHandler(ToFileHandler):
"""
Base class for export handlers using CSV file(s) as data target.
"""
class FromWuttaToCsv(
FromSqlalchemyToCsvHandlerMixin, FromWuttaHandler, ToCsvHandler
): # pylint: disable=too-many-ancestors
"""
Handler for Wutta (:term:`app database`) CSV export.
This uses :class:`FromSqlalchemyToCsvHandlerMixin` for most of the
heavy lifting.
"""
FromImporterBase = FromWutta
def get_source_model(self): # pylint: disable=empty-docstring
""" """
return self.app.model

View file

@ -0,0 +1,50 @@
# -*- coding: utf-8; -*-
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
# Wutta Framework is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# Wutta Framework is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# Wutta Framework. If not, see <http://www.gnu.org/licenses/>.
#
################################################################################
"""
Export Handlers
"""
from wuttasync.importing import ImportHandler, Orientation
class ExportHandler(ImportHandler):
"""
Generic base class for :term:`export handlers <import handler>`.
This is really just
:class:`~wuttasync.importing.handlers.ImportHandler` with the
orientation flipped.
"""
orientation = Orientation.EXPORT
"" # nb. suppress docs
class ToFileHandler(ExportHandler):
"""
Base class for export handlers which use output file(s) as the
data target.
Importers (exporters) used by this handler are generally assumed
to subclass :class:`~wuttasync.exporting.base.ToFile`.
"""

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -31,6 +31,8 @@ And some :term:`import handler` base classes:
* :class:`~wuttasync.importing.handlers.ImportHandler`
* :class:`~wuttasync.importing.handlers.FromFileHandler`
* :class:`~wuttasync.importing.handlers.FromSqlalchemyHandler`
* :class:`~wuttasync.importing.handlers.FromWuttaHandler`
* :class:`~wuttasync.importing.handlers.ToSqlalchemyHandler`
* :class:`~wuttasync.importing.handlers.ToWuttaHandler`
@ -38,16 +40,22 @@ And some :term:`importer` base classes:
* :class:`~wuttasync.importing.base.Importer`
* :class:`~wuttasync.importing.base.FromFile`
* :class:`~wuttasync.importing.base.FromSqlalchemy`
* :class:`~wuttasync.importing.base.FromWutta`
* :class:`~wuttasync.importing.base.ToSqlalchemy`
* :class:`~wuttasync.importing.model.ToWutta`
See also the :mod:`wuttasync.exporting` module.
"""
from .handlers import (
Orientation,
ImportHandler,
FromFileHandler,
FromSqlalchemyHandler,
FromWuttaHandler,
ToSqlalchemyHandler,
ToWuttaHandler,
)
from .base import Importer, FromFile, ToSqlalchemy
from .base import Importer, FromFile, FromSqlalchemy, FromWutta, ToSqlalchemy
from .model import ToWutta

View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024-2025 Lance Edgar
# Copyright © 2024-2026 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -1448,6 +1448,13 @@ class FromSqlalchemyMirror(FromSqlalchemy): # pylint: disable=abstract-method
return self.normalize_target_object(obj)
class FromWutta(FromSqlalchemy): # pylint: disable=abstract-method
"""
Base class for data importer/exporter which uses the Wutta ORM
(:term:`app database`) as data source.
"""
class ToSqlalchemy(Importer):
"""
Base class for importer/exporter which uses SQLAlchemy ORM on the

View file

@ -218,6 +218,8 @@ class ImportHandler( # pylint: disable=too-many-public-methods,too-many-instanc
* ``'importer'``
* ``'exporter'``
See also :attr:`actioning`.
"""
return f"{self.orientation.value}er"
@ -229,6 +231,8 @@ class ImportHandler( # pylint: disable=too-many-public-methods,too-many-instanc
* ``'importing'``
* ``'exporting'``
See also :attr:`actioner`.
"""
return f"{self.orientation.value}ing"

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8; -*-
import inspect
import sys
from unittest import TestCase
from unittest.mock import patch, Mock
@ -67,6 +68,75 @@ class TestImportCommandHandler(DataTestCase):
transaction_comment="hello world",
)
def test_run_missing_input(self):
handler = self.make_handler(
import_handler="wuttasync.importing.csv:FromCsvToWutta"
)
class Object:
def __init__(self, **kw):
self.__dict__.update(kw)
# fails without input_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={},
parent=Object(params={}),
)
try:
handler.run(ctx)
except RuntimeError:
pass
exit_.assert_called_once_with(1)
# runs with input_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={"input_file_path": self.tempdir},
parent=Object(
params={},
),
)
self.assertRaises(FileNotFoundError, handler.run, ctx)
exit_.assert_not_called()
def test_run_missing_output(self):
handler = self.make_handler(
import_handler="wuttasync.exporting.csv:FromWuttaToCsv"
)
class Object:
def __init__(self, **kw):
self.__dict__.update(kw)
# fails without output_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={},
parent=Object(params={}),
)
try:
handler.run(ctx)
except RuntimeError:
pass
exit_.assert_called_once_with(1)
# runs with output_file_path
with patch.object(sys, "exit") as exit_:
exit_.side_effect = RuntimeError
ctx = Object(
params={"output_file_path": self.tempdir},
parent=Object(
params={},
),
)
# self.assertRaises(FileNotFoundError, handler.run, ctx)
handler.run(ctx)
exit_.assert_not_called()
def test_list_models(self):
handler = self.make_handler(
import_handler="wuttasync.importing.csv:FromCsvToWutta"
@ -96,6 +166,23 @@ class TestImporterCommand(TestCase):
self.assertIn("dry_run", sig2.parameters)
class TestFileExporterCommand(TestCase):
def test_basic(self):
def myfunc(ctx, **kwargs):
pass
sig1 = inspect.signature(myfunc)
self.assertIn("kwargs", sig1.parameters)
self.assertNotIn("dry_run", sig1.parameters)
self.assertNotIn("output_file_path", sig1.parameters)
wrapt = mod.file_export_command(myfunc)
sig2 = inspect.signature(wrapt)
self.assertNotIn("kwargs", sig2.parameters)
self.assertIn("dry_run", sig2.parameters)
self.assertIn("output_file_path", sig2.parameters)
class TestFileImporterCommand(TestCase):
def test_basic(self):

View file

@ -0,0 +1,22 @@
# -*- coding: utf-8; -*-
from unittest import TestCase
from unittest.mock import MagicMock, patch
from wuttasync.cli import export_csv as mod, ImportCommandHandler
class TestExportCsv(TestCase):
def test_basic(self):
params = {
"models": [],
"create": True,
"update": True,
"delete": False,
"dry_run": True,
}
ctx = MagicMock(params=params)
with patch.object(ImportCommandHandler, "run") as run:
mod.export_csv(ctx)
run.assert_called_once_with(ctx)

View file

@ -0,0 +1,99 @@
# -*- coding: utf-8; -*-
from unittest.mock import patch
from wuttjamaican.testing import DataTestCase
from wuttasync.exporting import base as mod, ExportHandler
class TestToFile(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ExportHandler(self.config)
def make_exporter(self, **kwargs):
kwargs.setdefault("handler", self.handler)
return mod.ToFile(self.config, **kwargs)
def test_setup(self):
model = self.app.model
# output file is opened
exp = self.make_exporter(model_class=model.Setting)
self.assertFalse(exp.dry_run)
with patch.object(exp, "open_output_file") as open_output_file:
exp.setup()
open_output_file.assert_called_once_with()
# but not if in dry run mode
with patch.object(self.handler, "dry_run", new=True):
exp = self.make_exporter(model_class=model.Setting)
self.assertTrue(exp.dry_run)
with patch.object(exp, "open_output_file") as open_output_file:
exp.setup()
open_output_file.assert_not_called()
def test_teardown(self):
model = self.app.model
# output file is closed
exp = self.make_exporter(model_class=model.Setting)
self.assertFalse(exp.dry_run)
with patch.object(exp, "close_output_file") as close_output_file:
exp.teardown()
close_output_file.assert_called_once_with()
# but not if in dry run mode
with patch.object(self.handler, "dry_run", new=True):
exp = self.make_exporter(model_class=model.Setting)
self.assertTrue(exp.dry_run)
with patch.object(exp, "close_output_file") as close_output_file:
exp.teardown()
close_output_file.assert_not_called()
def test_get_output_file_path(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
# output path must be set
self.assertRaises(ValueError, exp.get_output_file_path)
# path is guessed from dir+filename
path1 = self.write_file("data1.txt", "")
exp.output_file_path = self.tempdir
exp.output_file_name = "data1.txt"
self.assertEqual(exp.get_output_file_path(), path1)
# path can be explicitly set
path2 = self.write_file("data2.txt", "")
exp.output_file_path = path2
self.assertEqual(exp.get_output_file_path(), path2)
def test_get_output_file_name(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
# name cannot be guessed
self.assertRaises(NotImplementedError, exp.get_output_file_name)
# name can be explicitly set
exp.output_file_name = "data.txt"
self.assertEqual(exp.get_output_file_name(), "data.txt")
def test_open_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
self.assertRaises(NotImplementedError, exp.open_output_file)
def test_close_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
path = self.write_file("data.txt", "")
with open(path, "wt") as f:
exp.output_file = f
with patch.object(f, "close") as close:
exp.close_output_file()
close.assert_called_once_with()

209
tests/exporting/test_csv.py Normal file
View file

@ -0,0 +1,209 @@
# -*- coding: utf-8; -*-
import csv
import io
from unittest.mock import patch
from wuttjamaican.testing import DataTestCase
from wuttasync.exporting import csv as mod, ExportHandler
from wuttasync.importing import FromWuttaHandler, FromWutta
class TestToCsv(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ExportHandler(self.config)
def make_exporter(self, **kwargs):
kwargs.setdefault("handler", self.handler)
kwargs.setdefault("output_file_path", self.tempdir)
return mod.ToCsv(self.config, **kwargs)
def test_get_output_file_name(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
# name can be guessed
self.assertEqual(exp.get_output_file_name(), "Setting.csv")
# name can be explicitly set
exp.output_file_name = "data.txt"
self.assertEqual(exp.get_output_file_name(), "data.txt")
def test_open_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
self.assertIsNone(exp.output_file)
self.assertIsNone(exp.output_writer)
exp.open_output_file()
try:
self.assertIsInstance(exp.output_file, io.TextIOBase)
self.assertIsInstance(exp.output_writer, csv.DictWriter)
finally:
exp.output_file.close()
def test_close_output_file(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
self.assertIsNone(exp.output_file)
self.assertIsNone(exp.output_writer)
exp.open_output_file()
self.assertIsNotNone(exp.output_file)
self.assertIsNotNone(exp.output_writer)
exp.close_output_file()
self.assertIsNone(exp.output_file)
self.assertIsNone(exp.output_writer)
def test_coerce_csv(self):
model = self.app.model
# string value
exp = self.make_exporter(model_class=model.Setting)
result = exp.coerce_csv({"name": "foo", "value": "bar"})
self.assertEqual(result, {"name": "foo", "value": "bar"})
# null value converts to empty string
result = exp.coerce_csv({"name": "foo", "value": None})
self.assertEqual(result, {"name": "foo", "value": ""})
# float value passed thru as-is
result = exp.coerce_csv({"name": "foo", "value": 12.34})
self.assertEqual(result, {"name": "foo", "value": 12.34})
self.assertIsInstance(result["value"], float)
def test_update_target_object(self):
model = self.app.model
exp = self.make_exporter(model_class=model.Setting)
exp.setup()
with patch.object(exp, "output_writer") as output_writer:
# writer is called for normal run
data = {"name": "foo", "value": "bar"}
exp.update_target_object(None, data)
output_writer.writerow.assert_called_once_with(data)
# but not called for dry run
output_writer.writerow.reset_mock()
with patch.object(self.handler, "dry_run", new=True):
exp.update_target_object(None, data)
output_writer.writerow.assert_not_called()
exp.teardown()
class MockMixinExporter(mod.FromSqlalchemyToCsvMixin, FromWutta, mod.ToCsv):
pass
class TestFromSqlalchemyToCsvMixin(DataTestCase):
def setUp(self):
self.setup_db()
self.handler = ExportHandler(self.config)
def make_exporter(self, **kwargs):
kwargs.setdefault("handler", self.handler)
return MockMixinExporter(self.config, **kwargs)
def test_model_title(self):
model = self.app.model
exp = self.make_exporter(source_model_class=model.Setting)
# default comes from model class
self.assertEqual(exp.get_model_title(), "Setting")
# but can override
exp.model_title = "Widget"
self.assertEqual(exp.get_model_title(), "Widget")
def test_get_simple_fields(self):
model = self.app.model
exp = self.make_exporter(source_model_class=model.Setting)
# default comes from model class
self.assertEqual(exp.get_simple_fields(), ["name", "value"])
# but can override
exp.simple_fields = ["name"]
self.assertEqual(exp.get_simple_fields(), ["name"])
# no default if no model class
exp = self.make_exporter()
self.assertEqual(exp.get_simple_fields(), [])
def test_normalize_source_object(self):
model = self.app.model
exp = self.make_exporter(source_model_class=model.Setting)
setting = model.Setting(name="foo", value="bar")
data = exp.normalize_source_object(setting)
self.assertEqual(data, {"name": "foo", "value": "bar"})
def test_make_object(self):
model = self.app.model
# normal
exp = self.make_exporter(source_model_class=model.Setting)
obj = exp.make_object()
self.assertIsInstance(obj, model.Setting)
# no model_class
exp = self.make_exporter()
self.assertRaises(TypeError, exp.make_object)
class MockMixinHandler(
mod.FromSqlalchemyToCsvHandlerMixin, FromWuttaHandler, mod.ToCsvHandler
):
FromImporterBase = FromWutta
class TestFromSqlalchemyToCsvHandlerMixin(DataTestCase):
def make_handler(self, **kwargs):
return MockMixinHandler(self.config, **kwargs)
def test_get_source_model(self):
with patch.object(
mod.FromSqlalchemyToCsvHandlerMixin, "define_importers", return_value={}
):
handler = self.make_handler()
self.assertRaises(NotImplementedError, handler.get_source_model)
def test_define_importers(self):
model = self.app.model
with patch.object(
mod.FromSqlalchemyToCsvHandlerMixin, "get_source_model", return_value=model
):
handler = self.make_handler()
importers = handler.define_importers()
self.assertIn("Setting", importers)
self.assertTrue(issubclass(importers["Setting"], FromWutta))
self.assertTrue(issubclass(importers["Setting"], mod.ToCsv))
self.assertIn("User", importers)
self.assertIn("Person", importers)
self.assertIn("Role", importers)
def test_make_importer_factory(self):
model = self.app.model
with patch.object(
mod.FromSqlalchemyToCsvHandlerMixin, "define_importers", return_value={}
):
handler = self.make_handler()
factory = handler.make_importer_factory(model.Setting, "Setting")
self.assertTrue(issubclass(factory, FromWutta))
self.assertTrue(issubclass(factory, mod.ToCsv))
class TestFromWuttaToCsv(DataTestCase):
def make_handler(self, **kwargs):
return mod.FromWuttaToCsv(self.config, **kwargs)
def test_get_source_model(self):
handler = self.make_handler()
self.assertIs(handler.get_source_model(), self.app.model)

View file

@ -0,0 +1,4 @@
# -*- coding: utf-8; -*-
# nothing to test yet really, just ensuring coverage
from wuttasync.exporting import handlers as mod