From 15b2cb07ba93381807ddc482e1a5e9c6da763ed0 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Fri, 6 Dec 2024 09:06:45 -0600 Subject: [PATCH] fix: add `--list-models` option for import/export commands also rename the command decorators for consistency --- docs/glossary.rst | 31 +++++++ docs/index.rst | 1 + src/wuttasync/cli/__init__.py | 7 +- src/wuttasync/cli/base.py | 149 +++++++++++++++++++++++++++----- src/wuttasync/cli/import_csv.py | 13 ++- src/wuttasync/importing/base.py | 2 + tests/cli/test_base.py | 55 +++++++++++- tests/cli/test_import_csv.py | 19 ++-- 8 files changed, 230 insertions(+), 47 deletions(-) create mode 100644 docs/glossary.rst diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 0000000..9bf2b30 --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,31 @@ +.. _glossary: + +Glossary +======== + +.. glossary:: + :sorted: + + import handler + This a type of :term:`handler` which is responsible for a + particular set of data import/export task(s). + + The import handler manages data connections and transactions, and + invokes one or more :term:`importers ` to process the + data. + + Note that "import/export handler" is the more proper term to use + here but it is often shortened to just "import handler" for + convenience. + + importer + In the context of WuttaSync, this refers to a type of object + which can process data for an import/export job, i.e. create, + update or delete records on the "target" based on the "source" + data it reads. + + See also :term:`import handler` which can "contain" one or more + importers. + + Note that "importer/exporter" is the more proper term to use here + but it is often shortened to just "importer" for convenience. diff --git a/docs/index.rst b/docs/index.rst index b8bf248..c8bc0cb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,6 +22,7 @@ database`, it may be used for any "source → target" data flow. :maxdepth: 2 :caption: Documentation + glossary narr/install narr/cli diff --git a/src/wuttasync/cli/__init__.py b/src/wuttasync/cli/__init__.py index 70de7ac..c77a4e2 100644 --- a/src/wuttasync/cli/__init__.py +++ b/src/wuttasync/cli/__init__.py @@ -25,11 +25,12 @@ WuttaSync - ``wutta`` subcommands This namespace exposes the following: -* :func:`~wuttasync.cli.base.importer_command()` -* :func:`~wuttasync.cli.base.file_importer_command()` +* :func:`~wuttasync.cli.base.import_command()` +* :func:`~wuttasync.cli.base.file_import_command()` +* :class:`~wuttasync.cli.base.ImportCommandHandler` """ -from .base import importer_command, file_importer_command +from .base import import_command, file_import_command, ImportCommandHandler # nb. must bring in all modules for discovery to work from . import import_csv diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index 008dd5b..f1268f3 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -25,6 +25,8 @@ """ import inspect +import logging +import sys from pathlib import Path from typing import List, Optional from typing_extensions import Annotated @@ -32,14 +34,118 @@ from typing_extensions import Annotated import makefun import typer +from wuttjamaican.app import GenericHandler +from wuttasync.importing import ImportHandler -def importer_command_template( + +log = logging.getLogger(__name__) + + +class ImportCommandHandler(GenericHandler): + """ + This is the :term:`handler` responsible for import/export command + line runs. + + Normally, the command (actually :term:`subcommand`) logic will + create this handler and call its :meth:`run()` method. + + This handler does not know how to import/export data, but it knows + how to make its :attr:`import_handler` do it. + + :param import_handler: During construction, caller can specify the + :attr:`import_handler` as any of: + + * import handler instance + * import handler factory (e.g. class) + * import handler spec (cf. :func:`~wuttjamaican:wuttjamaican.util.load_object()`) + + For example:: + + handler = ImportCommandHandler( + config, import_handler='wuttasync.importing.csv:FromCsvToWutta') + """ + + import_handler = None + """ + Reference to the :term:`import handler` instance, which is to be + invoked when command runs. See also :meth:`run()`. + """ + + def __init__(self, config, import_handler=None): + super().__init__(config) + + if import_handler: + if isinstance(import_handler, ImportHandler): + self.import_handler = import_handler + elif callable(import_handler): + self.import_handler = import_handler(self.config) + else: # spec + factory = self.app.load_object(import_handler) + self.import_handler = factory(self.config) + + def run(self, params, progress=None): + """ + Run the import/export job(s) based on command line params. + + This mostly just calls + :meth:`~wuttasync.importing.handlers.ImportHandler.process_data()` + for the :attr:`import_handler`. + + Unless ``--list-models`` was specified on the command line in + which case we do :meth:`list_models()` instead. + + :param params: Dict of params from command line. This must + include a ``'models'`` key, the rest are optional. + + :param progress: Optional progress indicator factory. + """ + + # maybe just list models and bail + if params.get('list_models'): + self.list_models(params) + return + + # otherwise process some data + kw = dict(params) + models = kw.pop('models') + log.debug("using handler: %s", self.import_handler.get_spec()) + # TODO: need to use all/default models if none specified + # (and should know models by now for logging purposes) + log.debug("running %s %s for: %s", + self.import_handler, + self.import_handler.orientation.value, + ', '.join(models)) + log.debug("params are: %s", kw) + self.import_handler.process_data(*models, **kw) + + def list_models(self, params): + """ + Query the :attr:`import_handler`'s supported target models and + print the info to stdout. + + This is what happens when command line has ``--list-models``. + """ + sys.stdout.write("ALL MODELS:\n") + sys.stdout.write("==============================\n") + for key in self.import_handler.importers: + sys.stdout.write(key) + sys.stdout.write("\n") + sys.stdout.write("==============================\n") + + +def import_command_template( # model keys models: Annotated[ Optional[List[str]], typer.Argument(help="Model(s) to process. Can specify one or more, " - "or omit to process all default models.")] = None, + "or omit to process default models.")] = None, + + # list models + list_models: Annotated[ + bool, + typer.Option('--list-models', '-l', + help="List available target models and exit.")] = False, # allow create? create: Annotated[ @@ -75,22 +181,22 @@ def importer_command_template( ): """ Stub function which provides a common param signature; used with - :func:`importer_command()`. + :func:`import_command()`. """ -def importer_command(fn): +def import_command(fn): """ Decorator for import/export commands. Adds common params based on - :func:`importer_command_template()`. + :func:`import_command_template()`. To use this, e.g. for ``poser import-foo`` command:: from poser.cli import poser_typer - from wuttasync.cli import importer_command + from wuttasync.cli import import_command, ImportCommandHandler @poser_typer.command() - @importer_command + @import_command def import_foo( ctx: typer.Context, **kwargs @@ -98,16 +204,15 @@ def importer_command(fn): \""" Import data from Foo API to Poser DB \""" - from poser.importing.foo import FromFooToPoser - config = ctx.parent.wutta_config - kw = dict(ctx.params) - models = kw.pop('models') - handler = FromFooToPoser(config) - handler.process_data(*models, **kw) + handler = ImportCommandHandler( + config, import_handler='poser.importing.foo:FromFooToPoser') + handler.run(ctx.params) + + See also :class:`ImportCommandHandler`. """ original_sig = inspect.signature(fn) - reference_sig = inspect.signature(importer_command_template) + reference_sig = inspect.signature(import_command_template) params = list(original_sig.parameters.values()) for i, param in enumerate(reference_sig.parameters.values()): @@ -120,7 +225,7 @@ def importer_command(fn): return makefun.create_function(final_sig, fn) -def file_importer_command_template( +def file_import_command_template( input_file_path: Annotated[ Path, typer.Option('--input-path', @@ -132,23 +237,23 @@ def file_importer_command_template( """ Stub function to provide signature for import/export commands which require input file. Used with - :func:`file_importer_command()`. + :func:`file_import_command()`. """ -def file_importer_command(fn): +def file_import_command(fn): """ Decorator for import/export commands which require input file. Adds common params based on - :func:`file_importer_command_template()`. + :func:`file_import_command_template()`. To use this, it's the same method as shown for - :func:`importer_command()` except in this case you would use the - ``file_importer_command`` decorator. + :func:`import_command()` except in this case you would use the + ``file_import_command`` decorator. """ original_sig = inspect.signature(fn) - plain_import_sig = inspect.signature(importer_command_template) - file_import_sig = inspect.signature(file_importer_command_template) + plain_import_sig = inspect.signature(import_command_template) + file_import_sig = inspect.signature(file_import_command_template) desired_params = ( list(plain_import_sig.parameters.values()) + list(file_import_sig.parameters.values())) diff --git a/src/wuttasync/cli/import_csv.py b/src/wuttasync/cli/import_csv.py index 7600d5f..50c2a83 100644 --- a/src/wuttasync/cli/import_csv.py +++ b/src/wuttasync/cli/import_csv.py @@ -30,11 +30,11 @@ import typer from wuttjamaican.cli import wutta_typer -from .base import file_importer_command +from .base import file_import_command, ImportCommandHandler @wutta_typer.command() -@file_importer_command +@file_import_command def import_csv( ctx: typer.Context, **kwargs @@ -42,10 +42,7 @@ def import_csv( """ Import data from CSV file(s) to Wutta DB """ - from wuttasync.importing.csv import FromCsvToWutta - config = ctx.parent.wutta_config - kw = dict(ctx.params) - models = kw.pop('models') - handler = FromCsvToWutta(config) - handler.process_data(*models, **kw) + handler = ImportCommandHandler( + config, import_handler='wuttasync.importing.csv:FromCsvToWutta') + handler.run(ctx.params) diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py index d46ae67..6ea84bb 100644 --- a/src/wuttasync/importing/base.py +++ b/src/wuttasync/importing/base.py @@ -373,6 +373,8 @@ class Importer: updated = [] deleted = [] + log.debug("using key fields: %s", ', '.join(self.get_keys())) + # get complete set of normalized source data if source_data is None: source_data = self.normalize_source_data(progress=progress) diff --git a/tests/cli/test_base.py b/tests/cli/test_base.py index b43f7d7..69af1b8 100644 --- a/tests/cli/test_base.py +++ b/tests/cli/test_base.py @@ -2,8 +2,59 @@ import inspect from unittest import TestCase +from unittest.mock import patch from wuttasync.cli import base as mod +from wuttjamaican.testing import DataTestCase + + +class TestImportCommandHandler(DataTestCase): + + def make_handler(self, **kwargs): + return mod.ImportCommandHandler(self.config, **kwargs) + + def test_import_handler(self): + + # none + handler = self.make_handler() + self.assertIsNone(handler.import_handler) + + FromCsvToWutta = self.app.load_object('wuttasync.importing.csv:FromCsvToWutta') + + # as spec + handler = self.make_handler(import_handler=FromCsvToWutta.get_spec()) + self.assertIsInstance(handler.import_handler, FromCsvToWutta) + + # as factory + handler = self.make_handler(import_handler=FromCsvToWutta) + self.assertIsInstance(handler.import_handler, FromCsvToWutta) + + # as instance + myhandler = FromCsvToWutta(self.config) + handler = self.make_handler(import_handler=myhandler) + self.assertIs(handler.import_handler, myhandler) + + def test_run(self): + handler = self.make_handler(import_handler='wuttasync.importing.csv:FromCsvToWutta') + + with patch.object(handler, 'list_models') as list_models: + handler.run({'list_models': True}) + list_models.assert_called_once_with({'list_models': True}) + + with patch.object(handler, 'import_handler') as import_handler: + handler.run({'models': []}) + import_handler.process_data.assert_called_once_with() + + def test_list_models(self): + handler = self.make_handler(import_handler='wuttasync.importing.csv:FromCsvToWutta') + + with patch.object(mod, 'sys') as sys: + handler.list_models({}) + # just test a few random things we expect to see + self.assertTrue(sys.stdout.write.has_call('ALL MODELS:\n')) + self.assertTrue(sys.stdout.write.has_call('Person')) + self.assertTrue(sys.stdout.write.has_call('User')) + self.assertTrue(sys.stdout.write.has_call('Upgrade')) class TestImporterCommand(TestCase): @@ -15,7 +66,7 @@ class TestImporterCommand(TestCase): sig1 = inspect.signature(myfunc) self.assertIn('kwargs', sig1.parameters) self.assertNotIn('dry_run', sig1.parameters) - wrapt = mod.importer_command(myfunc) + wrapt = mod.import_command(myfunc) sig2 = inspect.signature(wrapt) self.assertNotIn('kwargs', sig2.parameters) self.assertIn('dry_run', sig2.parameters) @@ -31,7 +82,7 @@ class TestFileImporterCommand(TestCase): self.assertIn('kwargs', sig1.parameters) self.assertNotIn('dry_run', sig1.parameters) self.assertNotIn('input_file_path', sig1.parameters) - wrapt = mod.file_importer_command(myfunc) + wrapt = mod.file_import_command(myfunc) sig2 = inspect.signature(wrapt) self.assertNotIn('kwargs', sig2.parameters) self.assertIn('dry_run', sig2.parameters) diff --git a/tests/cli/test_import_csv.py b/tests/cli/test_import_csv.py index a4371df..f856947 100644 --- a/tests/cli/test_import_csv.py +++ b/tests/cli/test_import_csv.py @@ -1,24 +1,19 @@ #-*- coding: utf-8; -*- -import os from unittest import TestCase from unittest.mock import MagicMock, patch -from wuttasync.cli import import_csv as mod -from wuttasync.importing.csv import FromCsvToWutta +from wuttasync.cli import import_csv as mod, ImportCommandHandler -here = os.path.dirname(__file__) -example_conf = os.path.join(here, 'example.conf') - class TestImportCsv(TestCase): def test_basic(self): - ctx = MagicMock(params={'models': [], - 'create': True, 'update': True, 'delete': False, - 'dry_run': True}) - with patch.object(FromCsvToWutta, 'process_data') as process_data: + params = {'models': [], + 'create': True, 'update': True, 'delete': False, + 'dry_run': True} + ctx = MagicMock(params=params) + with patch.object(ImportCommandHandler, 'run') as run: mod.import_csv(ctx) - process_data.assert_called_once_with(create=True, update=True, delete=False, - dry_run=True) + run.assert_called_once_with(params)