wutta/wuttasync
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: refactor per pylint; add to tox

Browse source
This commit is contained in:
Lance Edgar 2025-08-31 17:56:35 -05:00
parent 1aa70eba8b
commit e494bdd2b9
11 changed files with 77 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.pylintrc Normal file
View file

@ -0,0 +1,4 @@
# -*- mode: conf; -*-
[MESSAGES CONTROL]
disable=fixme

3
docs/index.rst
View file

@ -48,6 +48,9 @@ A "real-time sync" framework is also (eventually) planned, similar to
the one developed in the Rattail Project; the one developed in the Rattail Project;
cf. :doc:`rattail-manual:data/sync/index`. cf. :doc:`rattail-manual:data/sync/index`.
.. image:: https://img.shields.io/badge/linting-pylint-yellowgreen
:target: https://github.com/pylint-dev/pylint
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
:target: https://github.com/psf/black :target: https://github.com/psf/black

2
pyproject.toml
View file

@ -34,7 +34,7 @@ dependencies = [
[project.optional-dependencies] [project.optional-dependencies]
docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
tests = ["pytest-cov", "tox"] tests = ["pylint", "pytest", "pytest-cov", "tox"]
[project.entry-points."wutta.typer_imports"] [project.entry-points."wutta.typer_imports"]

3
src/wuttasync/_version.py
View file

@ -1,4 +1,7 @@
# -*- coding: utf-8; -*- # -*- coding: utf-8; -*-
"""
Package Version
"""
from importlib.metadata import version from importlib.metadata import version

10
src/wuttasync/cli/base.py
View file

@ -2,7 +2,7 @@
################################################################################ ################################################################################
# #
# WuttaSync -- Wutta Framework for data import/export and real-time sync # WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar # Copyright © 2024-2025 Lance Edgar
# #
# This file is part of Wutta Framework. # This file is part of Wutta Framework.
# #
@ -83,7 +83,7 @@ class ImportCommandHandler(GenericHandler):
factory = self.app.load_object(import_handler) factory = self.app.load_object(import_handler)
self.import_handler = factory(self.config) self.import_handler = factory(self.config)
def run(self, params, progress=None): def run(self, params, progress=None): # pylint: disable=unused-argument
""" """
Run the import/export job(s) based on command line params. Run the import/export job(s) based on command line params.
@ -120,7 +120,7 @@ class ImportCommandHandler(GenericHandler):
log.debug("params are: %s", kw) log.debug("params are: %s", kw)
self.import_handler.process_data(*models, **kw) self.import_handler.process_data(*models, **kw)
def list_models(self, params): def list_models(self, params): # pylint: disable=unused-argument
""" """
Query the :attr:`import_handler`'s supported target models and Query the :attr:`import_handler`'s supported target models and
print the info to stdout. print the info to stdout.
@ -135,7 +135,7 @@ class ImportCommandHandler(GenericHandler):
sys.stdout.write("==============================\n") sys.stdout.write("==============================\n")
def import_command_template( def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments
models: Annotated[ models: Annotated[
Optional[List[str]], Optional[List[str]],
typer.Argument( typer.Argument(
@ -270,7 +270,7 @@ def import_command(fn):
return makefun.create_function(final_sig, fn) return makefun.create_function(final_sig, fn)
def file_import_command_template( def file_import_command_template( # pylint: disable=unused-argument
input_file_path: Annotated[ input_file_path: Annotated[
Path, Path,
typer.Option( typer.Option(

6
src/wuttasync/cli/import_csv.py
View file

@ -2,7 +2,7 @@
################################################################################ ################################################################################
# #
# WuttaSync -- Wutta Framework for data import/export and real-time sync # WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar # Copyright © 2024-2025 Lance Edgar
# #
# This file is part of Wutta Framework. # This file is part of Wutta Framework.
# #
@ -24,8 +24,6 @@
See also: :ref:`wutta-import-csv` See also: :ref:`wutta-import-csv`
""" """
import os
import typer import typer
from wuttjamaican.cli import wutta_typer from wuttjamaican.cli import wutta_typer
@ -35,7 +33,7 @@ from .base import file_import_command, ImportCommandHandler
@wutta_typer.command() @wutta_typer.command()
@file_import_command @file_import_command
def import_csv(ctx: typer.Context, **kwargs): def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
""" """
Import data from CSV file(s) to Wutta DB Import data from CSV file(s) to Wutta DB
""" """

63
src/wuttasync/importing/base.py
View file

@ -2,7 +2,7 @@
################################################################################ ################################################################################
# #
# WuttaSync -- Wutta Framework for data import/export and real-time sync # WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar # Copyright © 2024-2025 Lance Edgar
# #
# This file is part of Wutta Framework. # This file is part of Wutta Framework.
# #
@ -23,6 +23,7 @@
""" """
Data Importer base class Data Importer base class
""" """
# pylint: disable=too-many-lines
import os import os
import logging import logging
@ -44,7 +45,7 @@ class ImportLimitReached(Exception):
""" """
class Importer: class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-methods
""" """
Base class for all data importers / exporters. Base class for all data importers / exporters.
@ -187,7 +188,7 @@ class Importer:
max_delete = None max_delete = None
max_total = None max_total = None
def __init__(self, config, **kwargs): def __init__(self, config, handler=None, model_class=None, **kwargs):
self.config = config self.config = config
self.app = self.config.get_app() self.app = self.config.get_app()
@ -201,6 +202,8 @@ class Importer:
"delete", kwargs.pop("allow_delete", self.allow_delete) "delete", kwargs.pop("allow_delete", self.allow_delete)
) )
self.handler = handler
self.model_class = model_class
self.__dict__.update(kwargs) self.__dict__.update(kwargs)
self.fields = self.get_fields() self.fields = self.get_fields()
@ -324,15 +327,15 @@ class Importer:
""" """
keys = None keys = None
# nb. prefer 'keys' but use 'key' as fallback # nb. prefer 'keys' but use 'key' as fallback
if hasattr(self, "keys"): if "keys" in self.__dict__:
keys = self.keys keys = self.__dict__["keys"]
elif hasattr(self, "key"): elif "key" in self.__dict__:
keys = self.key keys = self.__dict__["key"]
if keys: if keys:
if isinstance(keys, str): if isinstance(keys, str):
keys = self.config.parse_list(keys) keys = self.config.parse_list(keys)
# nb. save for next time # nb. save for next time
self.keys = keys self.__dict__["keys"] = keys
return keys return keys
return list(get_primary_keys(self.model_class)) return list(get_primary_keys(self.model_class))
@ -470,7 +473,7 @@ class Importer:
# cache the set of fields to use for diff checks # cache the set of fields to use for diff checks
fields = set(self.get_fields()) - set(self.get_keys()) fields = set(self.get_fields()) - set(self.get_keys())
def create_update(source_data, i): def create_update(source_data, i): # pylint: disable=unused-argument
# try to fetch target object per source key # try to fetch target object per source key
key = self.get_record_key(source_data) key = self.get_record_key(source_data)
@ -501,7 +504,7 @@ class Importer:
self.max_update, self.max_update,
) )
raise ImportLimitReached() raise ImportLimitReached()
elif ( if (
self.max_total self.max_total
and (len(created) + len(updated)) >= self.max_total and (len(created) + len(updated)) >= self.max_total
): ):
@ -532,7 +535,7 @@ class Importer:
self.max_create, self.max_create,
) )
raise ImportLimitReached() raise ImportLimitReached()
elif ( if (
self.max_total self.max_total
and (len(created) + len(updated)) >= self.max_total and (len(created) + len(updated)) >= self.max_total
): ):
@ -598,7 +601,7 @@ class Importer:
deletable = self.get_deletable_keys() - source_keys deletable = self.get_deletable_keys() - source_keys
log.debug("found %s records to delete", len(deletable)) log.debug("found %s records to delete", len(deletable))
def delete(key, i): def delete(key, i): # pylint: disable=unused-argument
cached = self.cached_target.pop(key) cached = self.cached_target.pop(key)
obj = cached["object"] obj = cached["object"]
@ -614,7 +617,7 @@ class Importer:
self.max_delete, self.max_delete,
) )
raise ImportLimitReached() raise ImportLimitReached()
elif self.max_total and (changes + len(deleted)) >= self.max_total: if self.max_total and (changes + len(deleted)) >= self.max_total:
log.warning( log.warning(
"max of %s *total changes* has been reached; stopping now", "max of %s *total changes* has been reached; stopping now",
self.max_total, self.max_total,
@ -711,7 +714,7 @@ class Importer:
source_objects = self.get_source_objects() source_objects = self.get_source_objects()
normalized = [] normalized = []
def normalize(obj, i): def normalize(obj, i): # pylint: disable=unused-argument
data = self.normalize_source_object_all(obj) data = self.normalize_source_object_all(obj)
if data: if data:
normalized.extend(data) normalized.extend(data)
@ -805,6 +808,7 @@ class Importer:
data = self.normalize_source_object(obj) data = self.normalize_source_object(obj)
if data: if data:
return [data] return [data]
return None
def normalize_source_object(self, obj): def normalize_source_object(self, obj):
""" """
@ -865,7 +869,7 @@ class Importer:
objects = self.get_target_objects(source_data=source_data) objects = self.get_target_objects(source_data=source_data)
cached = {} cached = {}
def cache(obj, i): def cache(obj, i): # pylint: disable=unused-argument
data = self.normalize_target_object(obj) data = self.normalize_target_object(obj)
if data: if data:
key = self.get_record_key(data) key = self.get_record_key(data)
@ -921,6 +925,7 @@ class Importer:
if self.caches_target and self.cached_target is not None: if self.caches_target and self.cached_target is not None:
cached = self.cached_target.get(key) cached = self.cached_target.get(key)
return cached["object"] if cached else None return cached["object"] if cached else None
return None
def normalize_target_object(self, obj): def normalize_target_object(self, obj):
""" """
@ -945,7 +950,7 @@ class Importer:
""" """
fields = self.get_fields() fields = self.get_fields()
fields = [f for f in self.get_simple_fields() if f in fields] fields = [f for f in self.get_simple_fields() if f in fields]
data = dict([(field, getattr(obj, field)) for field in fields]) data = {field: getattr(obj, field) for field in fields}
return data return data
def get_deletable_keys(self, progress=None): def get_deletable_keys(self, progress=None):
@ -970,7 +975,7 @@ class Importer:
keys = set() keys = set()
def check(key, i): def check(key, i): # pylint: disable=unused-argument
data = self.cached_target[key]["data"] data = self.cached_target[key]["data"]
obj = self.cached_target[key]["object"] obj = self.cached_target[key]["object"]
if self.can_delete_object(obj, data): if self.can_delete_object(obj, data):
@ -1000,11 +1005,12 @@ class Importer:
:returns: New object for the target side, or ``None``. :returns: New object for the target side, or ``None``.
""" """
if source_data.get("__ignoreme__"): if source_data.get("__ignoreme__"):
return return None
obj = self.make_empty_object(key) obj = self.make_empty_object(key)
if obj: if obj:
return self.update_target_object(obj, source_data) return self.update_target_object(obj, source_data)
return None
def make_empty_object(self, key): def make_empty_object(self, key):
""" """
@ -1072,11 +1078,11 @@ class Importer:
# object key(s) should already be populated # object key(s) should already be populated
continue continue
# elif field not in source_data: # if field not in source_data:
# # no source data for field # # no source data for field
# continue # continue
elif field in fields: if field in fields:
# field is eligible for update generally, so compare # field is eligible for update generally, so compare
# values between records # values between records
@ -1091,7 +1097,7 @@ class Importer:
return obj return obj
def can_delete_object(self, obj, data=None): def can_delete_object(self, obj, data=None): # pylint: disable=unused-argument
""" """
Should return true or false indicating whether the given Should return true or false indicating whether the given
object "can" be deleted. Default is to return true in all object "can" be deleted. Default is to return true in all
@ -1110,7 +1116,7 @@ class Importer:
""" """
return True return True
def delete_target_object(self, obj): def delete_target_object(self, obj): # pylint: disable=unused-argument
""" """
Delete the given raw object from the target side, and return Delete the given raw object from the target side, and return
true if successful. true if successful.
@ -1174,6 +1180,8 @@ class FromFile(Importer):
:meth:`close_input_file()`. :meth:`close_input_file()`.
""" """
input_file = None
def setup(self): def setup(self):
""" """
Open the input file. See also :meth:`open_input_file()`. Open the input file. See also :meth:`open_input_file()`.
@ -1267,6 +1275,8 @@ class ToSqlalchemy(Importer):
caches_target = True caches_target = True
"" # nb. suppress sphinx docs "" # nb. suppress sphinx docs
target_session = None
def get_target_object(self, key): def get_target_object(self, key):
""" """
Tries to fetch the object from target DB using ORM query. Tries to fetch the object from target DB using ORM query.
@ -1282,7 +1292,7 @@ class ToSqlalchemy(Importer):
try: try:
return query.one() return query.one()
except orm.exc.NoResultFound: except orm.exc.NoResultFound:
pass return None
def get_target_objects(self, source_data=None, progress=None): def get_target_objects(self, source_data=None, progress=None):
""" """
@ -1292,7 +1302,7 @@ class ToSqlalchemy(Importer):
query = self.get_target_query(source_data=source_data) query = self.get_target_query(source_data=source_data)
return query.all() return query.all()
def get_target_query(self, source_data=None): def get_target_query(self, source_data=None): # pylint: disable=unused-argument
""" """
Returns an ORM query suitable to fetch existing objects from Returns an ORM query suitable to fetch existing objects from
the target side. This is called from the target side. This is called from
@ -1300,7 +1310,7 @@ class ToSqlalchemy(Importer):
""" """
return self.target_session.query(self.model_class) return self.target_session.query(self.model_class)
def create_target_object(self, key, source_data): def create_target_object(self, key, source_data): # pylint: disable=empty-docstring
""" """ """ """
with self.target_session.no_autoflush: with self.target_session.no_autoflush:
obj = super().create_target_object(key, source_data) obj = super().create_target_object(key, source_data)
@ -1308,8 +1318,9 @@ class ToSqlalchemy(Importer):
# nb. add new object to target db session # nb. add new object to target db session
self.target_session.add(obj) self.target_session.add(obj)
return obj return obj
return None
def delete_target_object(self, obj): def delete_target_object(self, obj): # pylint: disable=empty-docstring
""" """ """ """
self.target_session.delete(obj) self.target_session.delete(obj)
return True return True

18
src/wuttasync/importing/csv.py
View file

@ -2,7 +2,7 @@
################################################################################ ################################################################################
# #
# WuttaSync -- Wutta Framework for data import/export and real-time sync # WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar # Copyright © 2024-2025 Lance Edgar
# #
# This file is part of Wutta Framework. # This file is part of Wutta Framework.
# #
@ -42,7 +42,7 @@ from .model import ToWutta
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class FromCsv(FromFile): class FromCsv(FromFile): # pylint: disable=abstract-method
""" """
Base class for importer/exporter using CSV file as data source. Base class for importer/exporter using CSV file as data source.
@ -61,6 +61,8 @@ class FromCsv(FromFile):
:class:`python:csv.DictReader` instance. :class:`python:csv.DictReader` instance.
""" """
input_reader = None
csv_encoding = "utf_8" csv_encoding = "utf_8"
""" """
Encoding used by the CSV input file. Encoding used by the CSV input file.
@ -104,7 +106,9 @@ class FromCsv(FromFile):
""" """
path = self.get_input_file_path() path = self.get_input_file_path()
log.debug("opening input file: %s", path) log.debug("opening input file: %s", path)
self.input_file = open(path, "rt", encoding=self.csv_encoding) self.input_file = open( # pylint: disable=consider-using-with
path, "rt", encoding=self.csv_encoding
)
self.input_reader = csv.DictReader(self.input_file) self.input_reader = csv.DictReader(self.input_file)
# nb. importer may have all supported fields by default, so # nb. importer may have all supported fields by default, so
@ -118,7 +122,7 @@ class FromCsv(FromFile):
self.input_file.close() self.input_file.close()
raise ValueError("input file has no recognized fields") raise ValueError("input file has no recognized fields")
def close_input_file(self): def close_input_file(self): # pylint: disable=empty-docstring
""" """ """ """
self.input_file.close() self.input_file.close()
del self.input_reader del self.input_reader
@ -136,7 +140,7 @@ class FromCsv(FromFile):
return list(self.input_reader) return list(self.input_reader)
class FromCsvToSqlalchemyMixin: class FromCsvToSqlalchemyMixin: # pylint: disable=too-few-public-methods
""" """
Mixin class for CSV → SQLAlchemy ORM :term:`importers <importer>`. Mixin class for CSV → SQLAlchemy ORM :term:`importers <importer>`.
@ -161,7 +165,7 @@ class FromCsvToSqlalchemyMixin:
if isinstance(attr.prop.columns[0].type, UUID): if isinstance(attr.prop.columns[0].type, UUID):
self.uuid_keys.append(field) self.uuid_keys.append(field)
def normalize_source_object(self, obj): def normalize_source_object(self, obj): # pylint: disable=empty-docstring
""" """ """ """
data = dict(obj) data = dict(obj)
@ -292,6 +296,6 @@ class FromCsvToWutta(FromCsvToSqlalchemyHandlerMixin, FromFileHandler, ToWuttaHa
ToImporterBase = ToWutta ToImporterBase = ToWutta
def get_target_model(self): def get_target_model(self): # pylint: disable=empty-docstring
""" """ """ """
return self.app.model return self.app.model

10
src/wuttasync/importing/handlers.py
View file

@ -2,7 +2,7 @@
################################################################################ ################################################################################
# #
# WuttaSync -- Wutta Framework for data import/export and real-time sync # WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar # Copyright © 2024-2025 Lance Edgar
# #
# This file is part of Wutta Framework. # This file is part of Wutta Framework.
# #
@ -275,7 +275,7 @@ class ImportHandler(GenericHandler):
) )
except: except:
# TODO: what should happen here? log.exception("what should happen here?") # TODO
raise raise
else: else:
@ -497,7 +497,7 @@ class ImportHandler(GenericHandler):
factory = self.importers[key] factory = self.importers[key]
return factory(self.config, **kwargs) return factory(self.config, **kwargs)
def get_importer_kwargs(self, key, **kwargs): def get_importer_kwargs(self, key, **kwargs): # pylint: disable=unused-argument
""" """
Returns a dict of kwargs to be used when construcing an Returns a dict of kwargs to be used when construcing an
importer/exporter with the given key. This is normally called importer/exporter with the given key. This is normally called
@ -522,7 +522,7 @@ class FromFileHandler(ImportHandler):
logic. logic.
""" """
def process_data(self, *keys, **kwargs): def process_data(self, *keys, **kwargs): # pylint: disable=empty-docstring
""" """ """ """
# interpret file vs. folder path # interpret file vs. folder path
@ -586,7 +586,7 @@ class ToSqlalchemyHandler(ImportHandler):
""" """
raise NotImplementedError raise NotImplementedError
def get_importer_kwargs(self, key, **kwargs): def get_importer_kwargs(self, key, **kwargs): # pylint: disable=empty-docstring
""" """ """ """
kwargs = super().get_importer_kwargs(key, **kwargs) kwargs = super().get_importer_kwargs(key, **kwargs)
kwargs.setdefault("target_session", self.target_session) kwargs.setdefault("target_session", self.target_session)

4
src/wuttasync/importing/wutta.py
View file

@ -2,7 +2,7 @@
################################################################################ ################################################################################
# #
# WuttaSync -- Wutta Framework for data import/export and real-time sync # WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar # Copyright © 2024-2025 Lance Edgar
# #
# This file is part of Wutta Framework. # This file is part of Wutta Framework.
# #
@ -36,7 +36,7 @@ class ToWuttaHandler(ToSqlalchemyHandler):
target_key = "wutta" target_key = "wutta"
"" # nb. suppress docs "" # nb. suppress docs
def get_target_title(self): def get_target_title(self): # pylint: disable=empty-docstring
""" """ """ """
# nb. we override parent to use app title as default # nb. we override parent to use app title as default
if hasattr(self, "target_title"): if hasattr(self, "target_title"):

4
tox.ini
View file

@ -6,6 +6,10 @@ envlist = py38, py39, py310, py311
extras = tests extras = tests
commands = pytest {posargs} commands = pytest {posargs}
[testenv:pylint]
basepython = python3.11
commands = pylint wuttasync
[testenv:coverage] [testenv:coverage]
basepython = python3.11 basepython = python3.11
commands = pytest --cov=wuttasync --cov-report=html --cov-fail-under=100 commands = pytest --cov=wuttasync --cov-report=html --cov-fail-under=100