fix: add --list-models option for import/export commands

also rename the command decorators for consistency

docs/glossary.rst

@@ -0,0 +1,31 @@
+.. _glossary:
+
+Glossary
+========
+
+.. glossary::
+   :sorted:
+
+   import handler
+     This a type of :term:`handler` which is responsible for a
+     particular set of data import/export task(s).
+
+     The import handler manages data connections and transactions, and
+     invokes one or more :term:`importers <importer>` to process the
+     data.
+
+     Note that "import/export handler" is the more proper term to use
+     here but it is often shortened to just "import handler" for
+     convenience.
+
+   importer
+     In the context of WuttaSync, this refers to a type of object
+     which can process data for an import/export job, i.e. create,
+     update or delete records on the "target" based on the "source"
+     data it reads.
+
+     See also :term:`import handler` which can "contain" one or more
+     importers.
+
+     Note that "importer/exporter" is the more proper term to use here
+     but it is often shortened to just "importer" for convenience.

docs/index.rst

@@ -22,6 +22,7 @@ database`, it may be used for any "source → target" data flow.
    :maxdepth: 2
    :caption: Documentation
 
+   glossary
    narr/install
    narr/cli
 

src/wuttasync/cli/__init__.py

@@ -25,11 +25,12 @@ WuttaSync - ``wutta`` subcommands
 
 This namespace exposes the following:
 
-* :func:`~wuttasync.cli.base.importer_command()`
+* :func:`~wuttasync.cli.base.import_command()`
-* :func:`~wuttasync.cli.base.file_importer_command()`
+* :func:`~wuttasync.cli.base.file_import_command()`
+* :class:`~wuttasync.cli.base.ImportCommandHandler`
 """
 
-from .base import importer_command, file_importer_command
+from .base import import_command, file_import_command, ImportCommandHandler
 
 # nb. must bring in all modules for discovery to work
 from . import import_csv

src/wuttasync/cli/base.py

@@ -25,6 +25,8 @@
 """
 
 import inspect
+import logging
+import sys
 from pathlib import Path
 from typing import List, Optional
 from typing_extensions import Annotated
@@ -32,14 +34,118 @@ from typing_extensions import Annotated
 import makefun
 import typer
 
+from wuttjamaican.app import GenericHandler
+from wuttasync.importing import ImportHandler
 
-def importer_command_template(
+
+log = logging.getLogger(__name__)
+
+
+class ImportCommandHandler(GenericHandler):
+    """
+    This is the :term:`handler` responsible for import/export command
+    line runs.
+
+    Normally, the command (actually :term:`subcommand`) logic will
+    create this handler and call its :meth:`run()` method.
+
+    This handler does not know how to import/export data, but it knows
+    how to make its :attr:`import_handler` do it.
+
+    :param import_handler: During construction, caller can specify the
+       :attr:`import_handler` as any of:
+
+       * import handler instance
+       * import handler factory (e.g. class)
+       * import handler spec (cf. :func:`~wuttjamaican:wuttjamaican.util.load_object()`)
+
+       For example::
+
+          handler = ImportCommandHandler(
+              config, import_handler='wuttasync.importing.csv:FromCsvToWutta')
+    """
+
+    import_handler = None
+    """
+    Reference to the :term:`import handler` instance, which is to be
+    invoked when command runs.  See also :meth:`run()`.
+    """
+
+    def __init__(self, config, import_handler=None):
+        super().__init__(config)
+
+        if import_handler:
+            if isinstance(import_handler, ImportHandler):
+                self.import_handler = import_handler
+            elif callable(import_handler):
+                self.import_handler = import_handler(self.config)
+            else: # spec
+                factory = self.app.load_object(import_handler)
+                self.import_handler = factory(self.config)
+
+    def run(self, params, progress=None):
+        """
+        Run the import/export job(s) based on command line params.
+
+        This mostly just calls
+        :meth:`~wuttasync.importing.handlers.ImportHandler.process_data()`
+        for the :attr:`import_handler`.
+
+        Unless ``--list-models`` was specified on the command line in
+        which case we do :meth:`list_models()` instead.
+
+        :param params: Dict of params from command line.  This must
+           include a ``'models'`` key, the rest are optional.
+
+        :param progress: Optional progress indicator factory.
+        """
+
+        # maybe just list models and bail
+        if params.get('list_models'):
+            self.list_models(params)
+            return
+
+        # otherwise process some data
+        kw = dict(params)
+        models = kw.pop('models')
+        log.debug("using handler: %s", self.import_handler.get_spec())
+        # TODO: need to use all/default models if none specified
+        # (and should know models by now for logging purposes)
+        log.debug("running %s %s for: %s",
+                  self.import_handler,
+                  self.import_handler.orientation.value,
+                  ', '.join(models))
+        log.debug("params are: %s", kw)
+        self.import_handler.process_data(*models, **kw)
+
+    def list_models(self, params):
+        """
+        Query the :attr:`import_handler`'s supported target models and
+        print the info to stdout.
+
+        This is what happens when command line has ``--list-models``.
+        """
+        sys.stdout.write("ALL MODELS:\n")
+        sys.stdout.write("==============================\n")
+        for key in self.import_handler.importers:
+            sys.stdout.write(key)
+            sys.stdout.write("\n")
+        sys.stdout.write("==============================\n")
+
+
+def import_command_template(
 
         # model keys
         models: Annotated[
             Optional[List[str]],
             typer.Argument(help="Model(s) to process.  Can specify one or more, "
-                           "or omit to process all default models.")] = None,
+                           "or omit to process default models.")] = None,
+
+        # list models
+        list_models: Annotated[
+            bool,
+            typer.Option('--list-models', '-l',
+                         help="List available target models and exit.")] = False,
 
         # allow create?
         create: Annotated[
@@ -75,22 +181,22 @@ def importer_command_template(
 ):
     """
     Stub function which provides a common param signature; used with
-    :func:`importer_command()`.
+    :func:`import_command()`.
     """
 
 
-def importer_command(fn):
+def import_command(fn):
     """
     Decorator for import/export commands.  Adds common params based on
-    :func:`importer_command_template()`.
+    :func:`import_command_template()`.
 
     To use this, e.g. for ``poser import-foo`` command::
 
        from poser.cli import poser_typer
-       from wuttasync.cli import importer_command
+       from wuttasync.cli import import_command, ImportCommandHandler
 
        @poser_typer.command()
-       @importer_command
+       @import_command
        def import_foo(
                ctx: typer.Context,
                **kwargs
@@ -98,16 +204,15 @@ def importer_command(fn):
            \"""
            Import data from Foo API to Poser DB
            \"""
-           from poser.importing.foo import FromFooToPoser
-
            config = ctx.parent.wutta_config
-           kw = dict(ctx.params)
+           handler = ImportCommandHandler(
-           models = kw.pop('models')
+               config, import_handler='poser.importing.foo:FromFooToPoser')
-           handler = FromFooToPoser(config)
+           handler.run(ctx.params)
-           handler.process_data(*models, **kw)
+
+    See also :class:`ImportCommandHandler`.
     """
     original_sig = inspect.signature(fn)
-    reference_sig = inspect.signature(importer_command_template)
+    reference_sig = inspect.signature(import_command_template)
 
     params = list(original_sig.parameters.values())
     for i, param in enumerate(reference_sig.parameters.values()):
@@ -120,7 +225,7 @@ def importer_command(fn):
     return makefun.create_function(final_sig, fn)
 
 
-def file_importer_command_template(
+def file_import_command_template(
         input_file_path: Annotated[
             Path,
             typer.Option('--input-path',
@@ -132,23 +237,23 @@ def file_importer_command_template(
     """
     Stub function to provide signature for import/export commands
     which require input file.  Used with
-    :func:`file_importer_command()`.
+    :func:`file_import_command()`.
     """
 
 
-def file_importer_command(fn):
+def file_import_command(fn):
     """
     Decorator for import/export commands which require input file.
     Adds common params based on
-    :func:`file_importer_command_template()`.
+    :func:`file_import_command_template()`.
 
     To use this, it's the same method as shown for
-    :func:`importer_command()` except in this case you would use the
+    :func:`import_command()` except in this case you would use the
-    ``file_importer_command`` decorator.
+    ``file_import_command`` decorator.
     """
     original_sig = inspect.signature(fn)
-    plain_import_sig = inspect.signature(importer_command_template)
+    plain_import_sig = inspect.signature(import_command_template)
-    file_import_sig = inspect.signature(file_importer_command_template)
+    file_import_sig = inspect.signature(file_import_command_template)
     desired_params = (
         list(plain_import_sig.parameters.values())
         + list(file_import_sig.parameters.values()))

src/wuttasync/cli/import_csv.py

@@ -30,11 +30,11 @@ import typer
 
 from wuttjamaican.cli import wutta_typer
 
-from .base import file_importer_command
+from .base import file_import_command, ImportCommandHandler
 
 
 @wutta_typer.command()
-@file_importer_command
+@file_import_command
 def import_csv(
         ctx: typer.Context,
         **kwargs
@@ -42,10 +42,7 @@ def import_csv(
     """
     Import data from CSV file(s) to Wutta DB
     """
-    from wuttasync.importing.csv import FromCsvToWutta
-
     config = ctx.parent.wutta_config
-    kw = dict(ctx.params)
+    handler = ImportCommandHandler(
-    models = kw.pop('models')
+        config, import_handler='wuttasync.importing.csv:FromCsvToWutta')
-    handler = FromCsvToWutta(config)
+    handler.run(ctx.params)
-    handler.process_data(*models, **kw)

src/wuttasync/importing/base.py

@@ -373,6 +373,8 @@ class Importer:
         updated = []
         deleted = []
 
+        log.debug("using key fields: %s", ', '.join(self.get_keys()))
+
         # get complete set of normalized source data
         if source_data is None:
             source_data = self.normalize_source_data(progress=progress)

tests/cli/test_base.py

@@ -2,8 +2,59 @@
 
 import inspect
 from unittest import TestCase
+from unittest.mock import patch
 
 from wuttasync.cli import base as mod
+from wuttjamaican.testing import DataTestCase
+
+
+class TestImportCommandHandler(DataTestCase):
+
+    def make_handler(self, **kwargs):
+        return mod.ImportCommandHandler(self.config, **kwargs)
+
+    def test_import_handler(self):
+
+        # none
+        handler = self.make_handler()
+        self.assertIsNone(handler.import_handler)
+
+        FromCsvToWutta = self.app.load_object('wuttasync.importing.csv:FromCsvToWutta')
+
+        # as spec
+        handler = self.make_handler(import_handler=FromCsvToWutta.get_spec())
+        self.assertIsInstance(handler.import_handler, FromCsvToWutta)
+
+        # as factory
+        handler = self.make_handler(import_handler=FromCsvToWutta)
+        self.assertIsInstance(handler.import_handler, FromCsvToWutta)
+
+        # as instance
+        myhandler = FromCsvToWutta(self.config)
+        handler = self.make_handler(import_handler=myhandler)
+        self.assertIs(handler.import_handler, myhandler)
+
+    def test_run(self):
+        handler = self.make_handler(import_handler='wuttasync.importing.csv:FromCsvToWutta')
+
+        with patch.object(handler, 'list_models') as list_models:
+            handler.run({'list_models': True})
+            list_models.assert_called_once_with({'list_models': True})
+
+        with patch.object(handler, 'import_handler') as import_handler:
+            handler.run({'models': []})
+            import_handler.process_data.assert_called_once_with()
+
+    def test_list_models(self):
+        handler = self.make_handler(import_handler='wuttasync.importing.csv:FromCsvToWutta')
+
+        with patch.object(mod, 'sys') as sys:
+            handler.list_models({})
+            # just test a few random things we expect to see
+            self.assertTrue(sys.stdout.write.has_call('ALL MODELS:\n'))
+            self.assertTrue(sys.stdout.write.has_call('Person'))
+            self.assertTrue(sys.stdout.write.has_call('User'))
+            self.assertTrue(sys.stdout.write.has_call('Upgrade'))
 
 
 class TestImporterCommand(TestCase):
@@ -15,7 +66,7 @@ class TestImporterCommand(TestCase):
         sig1 = inspect.signature(myfunc)
         self.assertIn('kwargs', sig1.parameters)
         self.assertNotIn('dry_run', sig1.parameters)
-        wrapt = mod.importer_command(myfunc)
+        wrapt = mod.import_command(myfunc)
         sig2 = inspect.signature(wrapt)
         self.assertNotIn('kwargs', sig2.parameters)
         self.assertIn('dry_run', sig2.parameters)
@@ -31,7 +82,7 @@ class TestFileImporterCommand(TestCase):
         self.assertIn('kwargs', sig1.parameters)
         self.assertNotIn('dry_run', sig1.parameters)
         self.assertNotIn('input_file_path', sig1.parameters)
-        wrapt = mod.file_importer_command(myfunc)
+        wrapt = mod.file_import_command(myfunc)
         sig2 = inspect.signature(wrapt)
         self.assertNotIn('kwargs', sig2.parameters)
         self.assertIn('dry_run', sig2.parameters)

tests/cli/test_import_csv.py

@@ -1,24 +1,19 @@
 #-*- coding: utf-8; -*-
 
-import os
 from unittest import TestCase
 from unittest.mock import MagicMock, patch
 
-from wuttasync.cli import import_csv as mod
+from wuttasync.cli import import_csv as mod, ImportCommandHandler
-from wuttasync.importing.csv import FromCsvToWutta
 
 
-here = os.path.dirname(__file__)
-example_conf = os.path.join(here, 'example.conf')
-
 
 class TestImportCsv(TestCase):
 
     def test_basic(self):
-        ctx = MagicMock(params={'models': [],
+        params = {'models': [],
-                                'create': True, 'update': True, 'delete': False,
+                  'create': True, 'update': True, 'delete': False,
-                                'dry_run': True})
+                  'dry_run': True}
-        with patch.object(FromCsvToWutta, 'process_data') as process_data:
+        ctx = MagicMock(params=params)
+        with patch.object(ImportCommandHandler, 'run') as run:
             mod.import_csv(ctx)
-            process_data.assert_called_once_with(create=True, update=True, delete=False,
+            run.assert_called_once_with(params)
-                                                 dry_run=True)