wutta/wuttasync
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: refactor per pylint; add to tox

Browse source
This commit is contained in:
Lance Edgar 2025-08-31 17:56:35 -05:00
parent 1aa70eba8b
commit e494bdd2b9
11 changed files with 77 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.pylintrc Normal file
View file

@ -0,0 +1,4 @@
# -*- mode: conf; -*-
[MESSAGES CONTROL]
disable=fixme

3
docs/index.rst
View file

@ -48,6 +48,9 @@ A "real-time sync" framework is also (eventually) planned, similar to
the one developed in the Rattail Project;
cf. :doc:`rattail-manual:data/sync/index`.
.. image:: https://img.shields.io/badge/linting-pylint-yellowgreen
:target: https://github.com/pylint-dev/pylint
.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
:target: https://github.com/psf/black

2
pyproject.toml
View file

@ -34,7 +34,7 @@ dependencies = [
[project.optional-dependencies]
docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"]
tests = ["pytest-cov", "tox"]
tests = ["pylint", "pytest", "pytest-cov", "tox"]
[project.entry-points."wutta.typer_imports"]

3
src/wuttasync/_version.py
View file

@ -1,4 +1,7 @@
# -*- coding: utf-8; -*-
"""
Package Version
"""
from importlib.metadata import version

10
src/wuttasync/cli/base.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar
# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -83,7 +83,7 @@ class ImportCommandHandler(GenericHandler):
factory = self.app.load_object(import_handler)
self.import_handler = factory(self.config)
def run(self, params, progress=None):
def run(self, params, progress=None): # pylint: disable=unused-argument
"""
Run the import/export job(s) based on command line params.
@ -120,7 +120,7 @@ class ImportCommandHandler(GenericHandler):
log.debug("params are: %s", kw)
self.import_handler.process_data(*models, **kw)
def list_models(self, params):
def list_models(self, params): # pylint: disable=unused-argument
"""
Query the :attr:`import_handler`'s supported target models and
print the info to stdout.
@ -135,7 +135,7 @@ class ImportCommandHandler(GenericHandler):
sys.stdout.write("==============================\n")
def import_command_template(
def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments
models: Annotated[
Optional[List[str]],
typer.Argument(
@ -270,7 +270,7 @@ def import_command(fn):
return makefun.create_function(final_sig, fn)
def file_import_command_template(
def file_import_command_template( # pylint: disable=unused-argument
input_file_path: Annotated[
Path,
typer.Option(

6
src/wuttasync/cli/import_csv.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar
# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -24,8 +24,6 @@
See also: :ref:`wutta-import-csv`
"""
import os
import typer
from wuttjamaican.cli import wutta_typer
@ -35,7 +33,7 @@ from .base import file_import_command, ImportCommandHandler
@wutta_typer.command()
@file_import_command
def import_csv(ctx: typer.Context, **kwargs):
def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument
"""
Import data from CSV file(s) to Wutta DB
"""

63
src/wuttasync/importing/base.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar
# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -23,6 +23,7 @@
"""
Data Importer base class
"""
# pylint: disable=too-many-lines
import os
import logging
@ -44,7 +45,7 @@ class ImportLimitReached(Exception):
"""
class Importer:
class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-methods
"""
Base class for all data importers / exporters.
@ -187,7 +188,7 @@ class Importer:
max_delete = None
max_total = None
def __init__(self, config, **kwargs):
def __init__(self, config, handler=None, model_class=None, **kwargs):
self.config = config
self.app = self.config.get_app()
@ -201,6 +202,8 @@ class Importer:
"delete", kwargs.pop("allow_delete", self.allow_delete)
)
self.handler = handler
self.model_class = model_class
self.__dict__.update(kwargs)
self.fields = self.get_fields()
@ -324,15 +327,15 @@ class Importer:
"""
keys = None
# nb. prefer 'keys' but use 'key' as fallback
if hasattr(self, "keys"):
keys = self.keys
elif hasattr(self, "key"):
keys = self.key
if "keys" in self.__dict__:
keys = self.__dict__["keys"]
elif "key" in self.__dict__:
keys = self.__dict__["key"]
if keys:
if isinstance(keys, str):
keys = self.config.parse_list(keys)
# nb. save for next time
self.keys = keys
self.__dict__["keys"] = keys
return keys
return list(get_primary_keys(self.model_class))
@ -470,7 +473,7 @@ class Importer:
# cache the set of fields to use for diff checks
fields = set(self.get_fields()) - set(self.get_keys())
def create_update(source_data, i):
def create_update(source_data, i): # pylint: disable=unused-argument
# try to fetch target object per source key
key = self.get_record_key(source_data)
@ -501,7 +504,7 @@ class Importer:
self.max_update,
)
raise ImportLimitReached()
elif (
if (
self.max_total
and (len(created) + len(updated)) >= self.max_total
):
@ -532,7 +535,7 @@ class Importer:
self.max_create,
)
raise ImportLimitReached()
elif (
if (
self.max_total
and (len(created) + len(updated)) >= self.max_total
):
@ -598,7 +601,7 @@ class Importer:
deletable = self.get_deletable_keys() - source_keys
log.debug("found %s records to delete", len(deletable))
def delete(key, i):
def delete(key, i): # pylint: disable=unused-argument
cached = self.cached_target.pop(key)
obj = cached["object"]
@ -614,7 +617,7 @@ class Importer:
self.max_delete,
)
raise ImportLimitReached()
elif self.max_total and (changes + len(deleted)) >= self.max_total:
if self.max_total and (changes + len(deleted)) >= self.max_total:
log.warning(
"max of %s *total changes* has been reached; stopping now",
self.max_total,
@ -711,7 +714,7 @@ class Importer:
source_objects = self.get_source_objects()
normalized = []
def normalize(obj, i):
def normalize(obj, i): # pylint: disable=unused-argument
data = self.normalize_source_object_all(obj)
if data:
normalized.extend(data)
@ -805,6 +808,7 @@ class Importer:
data = self.normalize_source_object(obj)
if data:
return [data]
return None
def normalize_source_object(self, obj):
"""
@ -865,7 +869,7 @@ class Importer:
objects = self.get_target_objects(source_data=source_data)
cached = {}
def cache(obj, i):
def cache(obj, i): # pylint: disable=unused-argument
data = self.normalize_target_object(obj)
if data:
key = self.get_record_key(data)
@ -921,6 +925,7 @@ class Importer:
if self.caches_target and self.cached_target is not None:
cached = self.cached_target.get(key)
return cached["object"] if cached else None
return None
def normalize_target_object(self, obj):
"""
@ -945,7 +950,7 @@ class Importer:
"""
fields = self.get_fields()
fields = [f for f in self.get_simple_fields() if f in fields]
data = dict([(field, getattr(obj, field)) for field in fields])
data = {field: getattr(obj, field) for field in fields}
return data
def get_deletable_keys(self, progress=None):
@ -970,7 +975,7 @@ class Importer:
keys = set()
def check(key, i):
def check(key, i): # pylint: disable=unused-argument
data = self.cached_target[key]["data"]
obj = self.cached_target[key]["object"]
if self.can_delete_object(obj, data):
@ -1000,11 +1005,12 @@ class Importer:
:returns: New object for the target side, or ``None``.
"""
if source_data.get("__ignoreme__"):
return
return None
obj = self.make_empty_object(key)
if obj:
return self.update_target_object(obj, source_data)
return None
def make_empty_object(self, key):
"""
@ -1072,11 +1078,11 @@ class Importer:
# object key(s) should already be populated
continue
# elif field not in source_data:
# if field not in source_data:
# # no source data for field
# continue
elif field in fields:
if field in fields:
# field is eligible for update generally, so compare
# values between records
@ -1091,7 +1097,7 @@ class Importer:
return obj
def can_delete_object(self, obj, data=None):
def can_delete_object(self, obj, data=None): # pylint: disable=unused-argument
"""
Should return true or false indicating whether the given
object "can" be deleted. Default is to return true in all
@ -1110,7 +1116,7 @@ class Importer:
"""
return True
def delete_target_object(self, obj):
def delete_target_object(self, obj): # pylint: disable=unused-argument
"""
Delete the given raw object from the target side, and return
true if successful.
@ -1174,6 +1180,8 @@ class FromFile(Importer):
:meth:`close_input_file()`.
"""
input_file = None
def setup(self):
"""
Open the input file. See also :meth:`open_input_file()`.
@ -1267,6 +1275,8 @@ class ToSqlalchemy(Importer):
caches_target = True
"" # nb. suppress sphinx docs
target_session = None
def get_target_object(self, key):
"""
Tries to fetch the object from target DB using ORM query.
@ -1282,7 +1292,7 @@ class ToSqlalchemy(Importer):
try:
return query.one()
except orm.exc.NoResultFound:
pass
return None
def get_target_objects(self, source_data=None, progress=None):
"""
@ -1292,7 +1302,7 @@ class ToSqlalchemy(Importer):
query = self.get_target_query(source_data=source_data)
return query.all()
def get_target_query(self, source_data=None):
def get_target_query(self, source_data=None): # pylint: disable=unused-argument
"""
Returns an ORM query suitable to fetch existing objects from
the target side. This is called from
@ -1300,7 +1310,7 @@ class ToSqlalchemy(Importer):
"""
return self.target_session.query(self.model_class)
def create_target_object(self, key, source_data):
def create_target_object(self, key, source_data): # pylint: disable=empty-docstring
""" """
with self.target_session.no_autoflush:
obj = super().create_target_object(key, source_data)
@ -1308,8 +1318,9 @@ class ToSqlalchemy(Importer):
# nb. add new object to target db session
self.target_session.add(obj)
return obj
return None
def delete_target_object(self, obj):
def delete_target_object(self, obj): # pylint: disable=empty-docstring
""" """
self.target_session.delete(obj)
return True

18
src/wuttasync/importing/csv.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar
# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -42,7 +42,7 @@ from .model import ToWutta
log = logging.getLogger(__name__)
class FromCsv(FromFile):
class FromCsv(FromFile): # pylint: disable=abstract-method
"""
Base class for importer/exporter using CSV file as data source.
@ -61,6 +61,8 @@ class FromCsv(FromFile):
:class:`python:csv.DictReader` instance.
"""
input_reader = None
csv_encoding = "utf_8"
"""
Encoding used by the CSV input file.
@ -104,7 +106,9 @@ class FromCsv(FromFile):
"""
path = self.get_input_file_path()
log.debug("opening input file: %s", path)
self.input_file = open(path, "rt", encoding=self.csv_encoding)
self.input_file = open( # pylint: disable=consider-using-with
path, "rt", encoding=self.csv_encoding
)
self.input_reader = csv.DictReader(self.input_file)
# nb. importer may have all supported fields by default, so
@ -118,7 +122,7 @@ class FromCsv(FromFile):
self.input_file.close()
raise ValueError("input file has no recognized fields")
def close_input_file(self):
def close_input_file(self): # pylint: disable=empty-docstring
""" """
self.input_file.close()
del self.input_reader
@ -136,7 +140,7 @@ class FromCsv(FromFile):
return list(self.input_reader)
class FromCsvToSqlalchemyMixin:
class FromCsvToSqlalchemyMixin: # pylint: disable=too-few-public-methods
"""
Mixin class for CSV → SQLAlchemy ORM :term:`importers <importer>`.
@ -161,7 +165,7 @@ class FromCsvToSqlalchemyMixin:
if isinstance(attr.prop.columns[0].type, UUID):
self.uuid_keys.append(field)
def normalize_source_object(self, obj):
def normalize_source_object(self, obj): # pylint: disable=empty-docstring
""" """
data = dict(obj)
@ -292,6 +296,6 @@ class FromCsvToWutta(FromCsvToSqlalchemyHandlerMixin, FromFileHandler, ToWuttaHa
ToImporterBase = ToWutta
def get_target_model(self):
def get_target_model(self): # pylint: disable=empty-docstring
""" """
return self.app.model

10
src/wuttasync/importing/handlers.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar
# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -275,7 +275,7 @@ class ImportHandler(GenericHandler):
)
except:
# TODO: what should happen here?
log.exception("what should happen here?") # TODO
raise
else:
@ -497,7 +497,7 @@ class ImportHandler(GenericHandler):
factory = self.importers[key]
return factory(self.config, **kwargs)
def get_importer_kwargs(self, key, **kwargs):
def get_importer_kwargs(self, key, **kwargs): # pylint: disable=unused-argument
"""
Returns a dict of kwargs to be used when construcing an
importer/exporter with the given key. This is normally called
@ -522,7 +522,7 @@ class FromFileHandler(ImportHandler):
logic.
"""
def process_data(self, *keys, **kwargs):
def process_data(self, *keys, **kwargs): # pylint: disable=empty-docstring
""" """
# interpret file vs. folder path
@ -586,7 +586,7 @@ class ToSqlalchemyHandler(ImportHandler):
"""
raise NotImplementedError
def get_importer_kwargs(self, key, **kwargs):
def get_importer_kwargs(self, key, **kwargs): # pylint: disable=empty-docstring
""" """
kwargs = super().get_importer_kwargs(key, **kwargs)
kwargs.setdefault("target_session", self.target_session)

4
src/wuttasync/importing/wutta.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# WuttaSync -- Wutta Framework for data import/export and real-time sync
# Copyright © 2024 Lance Edgar
# Copyright © 2024-2025 Lance Edgar
#
# This file is part of Wutta Framework.
#
@ -36,7 +36,7 @@ class ToWuttaHandler(ToSqlalchemyHandler):
target_key = "wutta"
"" # nb. suppress docs
def get_target_title(self):
def get_target_title(self): # pylint: disable=empty-docstring
""" """
# nb. we override parent to use app title as default
if hasattr(self, "target_title"):

4
tox.ini
View file

@ -6,6 +6,10 @@ envlist = py38, py39, py310, py311
extras = tests
commands = pytest {posargs}
[testenv:pylint]
basepython = python3.11
commands = pylint wuttasync
[testenv:coverage]
basepython = python3.11
commands = pytest --cov=wuttasync --cov-report=html --cov-fail-under=100