From e494bdd2b9f6b4079888cc6fbfa7464ae327cfbd Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Sun, 31 Aug 2025 17:56:35 -0500 Subject: [PATCH] fix: refactor per pylint; add to tox --- .pylintrc | 4 ++ docs/index.rst | 3 ++ pyproject.toml | 2 +- src/wuttasync/_version.py | 3 ++ src/wuttasync/cli/base.py | 10 ++--- src/wuttasync/cli/import_csv.py | 6 +-- src/wuttasync/importing/base.py | 63 +++++++++++++++++------------ src/wuttasync/importing/csv.py | 18 +++++---- src/wuttasync/importing/handlers.py | 10 ++--- src/wuttasync/importing/wutta.py | 4 +- tox.ini | 4 ++ 11 files changed, 77 insertions(+), 50 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..7eb5e2c --- /dev/null +++ b/.pylintrc @@ -0,0 +1,4 @@ +# -*- mode: conf; -*- + +[MESSAGES CONTROL] +disable=fixme diff --git a/docs/index.rst b/docs/index.rst index 42e04b1..9eb2d93 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,9 @@ A "real-time sync" framework is also (eventually) planned, similar to the one developed in the Rattail Project; cf. :doc:`rattail-manual:data/sync/index`. +.. image:: https://img.shields.io/badge/linting-pylint-yellowgreen + :target: https://github.com/pylint-dev/pylint + .. image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/psf/black diff --git a/pyproject.toml b/pyproject.toml index fc09dc9..a48b949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ [project.optional-dependencies] docs = ["Sphinx", "enum-tools[sphinx]", "furo", "sphinxcontrib-programoutput"] -tests = ["pytest-cov", "tox"] +tests = ["pylint", "pytest", "pytest-cov", "tox"] [project.entry-points."wutta.typer_imports"] diff --git a/src/wuttasync/_version.py b/src/wuttasync/_version.py index 690bd4f..4881c5c 100644 --- a/src/wuttasync/_version.py +++ b/src/wuttasync/_version.py @@ -1,4 +1,7 @@ # -*- coding: utf-8; -*- +""" +Package Version +""" from importlib.metadata import version diff --git a/src/wuttasync/cli/base.py b/src/wuttasync/cli/base.py index f9198d7..08fa4f5 100644 --- a/src/wuttasync/cli/base.py +++ b/src/wuttasync/cli/base.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -83,7 +83,7 @@ class ImportCommandHandler(GenericHandler): factory = self.app.load_object(import_handler) self.import_handler = factory(self.config) - def run(self, params, progress=None): + def run(self, params, progress=None): # pylint: disable=unused-argument """ Run the import/export job(s) based on command line params. @@ -120,7 +120,7 @@ class ImportCommandHandler(GenericHandler): log.debug("params are: %s", kw) self.import_handler.process_data(*models, **kw) - def list_models(self, params): + def list_models(self, params): # pylint: disable=unused-argument """ Query the :attr:`import_handler`'s supported target models and print the info to stdout. @@ -135,7 +135,7 @@ class ImportCommandHandler(GenericHandler): sys.stdout.write("==============================\n") -def import_command_template( +def import_command_template( # pylint: disable=unused-argument,too-many-arguments,too-many-positional-arguments models: Annotated[ Optional[List[str]], typer.Argument( @@ -270,7 +270,7 @@ def import_command(fn): return makefun.create_function(final_sig, fn) -def file_import_command_template( +def file_import_command_template( # pylint: disable=unused-argument input_file_path: Annotated[ Path, typer.Option( diff --git a/src/wuttasync/cli/import_csv.py b/src/wuttasync/cli/import_csv.py index 0b8716c..d3c8047 100644 --- a/src/wuttasync/cli/import_csv.py +++ b/src/wuttasync/cli/import_csv.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -24,8 +24,6 @@ See also: :ref:`wutta-import-csv` """ -import os - import typer from wuttjamaican.cli import wutta_typer @@ -35,7 +33,7 @@ from .base import file_import_command, ImportCommandHandler @wutta_typer.command() @file_import_command -def import_csv(ctx: typer.Context, **kwargs): +def import_csv(ctx: typer.Context, **kwargs): # pylint: disable=unused-argument """ Import data from CSV file(s) to Wutta DB """ diff --git a/src/wuttasync/importing/base.py b/src/wuttasync/importing/base.py index c9fcc9a..e8aa523 100644 --- a/src/wuttasync/importing/base.py +++ b/src/wuttasync/importing/base.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -23,6 +23,7 @@ """ Data Importer base class """ +# pylint: disable=too-many-lines import os import logging @@ -44,7 +45,7 @@ class ImportLimitReached(Exception): """ -class Importer: +class Importer: # pylint: disable=too-many-instance-attributes,too-many-public-methods """ Base class for all data importers / exporters. @@ -187,7 +188,7 @@ class Importer: max_delete = None max_total = None - def __init__(self, config, **kwargs): + def __init__(self, config, handler=None, model_class=None, **kwargs): self.config = config self.app = self.config.get_app() @@ -201,6 +202,8 @@ class Importer: "delete", kwargs.pop("allow_delete", self.allow_delete) ) + self.handler = handler + self.model_class = model_class self.__dict__.update(kwargs) self.fields = self.get_fields() @@ -324,15 +327,15 @@ class Importer: """ keys = None # nb. prefer 'keys' but use 'key' as fallback - if hasattr(self, "keys"): - keys = self.keys - elif hasattr(self, "key"): - keys = self.key + if "keys" in self.__dict__: + keys = self.__dict__["keys"] + elif "key" in self.__dict__: + keys = self.__dict__["key"] if keys: if isinstance(keys, str): keys = self.config.parse_list(keys) # nb. save for next time - self.keys = keys + self.__dict__["keys"] = keys return keys return list(get_primary_keys(self.model_class)) @@ -470,7 +473,7 @@ class Importer: # cache the set of fields to use for diff checks fields = set(self.get_fields()) - set(self.get_keys()) - def create_update(source_data, i): + def create_update(source_data, i): # pylint: disable=unused-argument # try to fetch target object per source key key = self.get_record_key(source_data) @@ -501,7 +504,7 @@ class Importer: self.max_update, ) raise ImportLimitReached() - elif ( + if ( self.max_total and (len(created) + len(updated)) >= self.max_total ): @@ -532,7 +535,7 @@ class Importer: self.max_create, ) raise ImportLimitReached() - elif ( + if ( self.max_total and (len(created) + len(updated)) >= self.max_total ): @@ -598,7 +601,7 @@ class Importer: deletable = self.get_deletable_keys() - source_keys log.debug("found %s records to delete", len(deletable)) - def delete(key, i): + def delete(key, i): # pylint: disable=unused-argument cached = self.cached_target.pop(key) obj = cached["object"] @@ -614,7 +617,7 @@ class Importer: self.max_delete, ) raise ImportLimitReached() - elif self.max_total and (changes + len(deleted)) >= self.max_total: + if self.max_total and (changes + len(deleted)) >= self.max_total: log.warning( "max of %s *total changes* has been reached; stopping now", self.max_total, @@ -711,7 +714,7 @@ class Importer: source_objects = self.get_source_objects() normalized = [] - def normalize(obj, i): + def normalize(obj, i): # pylint: disable=unused-argument data = self.normalize_source_object_all(obj) if data: normalized.extend(data) @@ -805,6 +808,7 @@ class Importer: data = self.normalize_source_object(obj) if data: return [data] + return None def normalize_source_object(self, obj): """ @@ -865,7 +869,7 @@ class Importer: objects = self.get_target_objects(source_data=source_data) cached = {} - def cache(obj, i): + def cache(obj, i): # pylint: disable=unused-argument data = self.normalize_target_object(obj) if data: key = self.get_record_key(data) @@ -921,6 +925,7 @@ class Importer: if self.caches_target and self.cached_target is not None: cached = self.cached_target.get(key) return cached["object"] if cached else None + return None def normalize_target_object(self, obj): """ @@ -945,7 +950,7 @@ class Importer: """ fields = self.get_fields() fields = [f for f in self.get_simple_fields() if f in fields] - data = dict([(field, getattr(obj, field)) for field in fields]) + data = {field: getattr(obj, field) for field in fields} return data def get_deletable_keys(self, progress=None): @@ -970,7 +975,7 @@ class Importer: keys = set() - def check(key, i): + def check(key, i): # pylint: disable=unused-argument data = self.cached_target[key]["data"] obj = self.cached_target[key]["object"] if self.can_delete_object(obj, data): @@ -1000,11 +1005,12 @@ class Importer: :returns: New object for the target side, or ``None``. """ if source_data.get("__ignoreme__"): - return + return None obj = self.make_empty_object(key) if obj: return self.update_target_object(obj, source_data) + return None def make_empty_object(self, key): """ @@ -1072,11 +1078,11 @@ class Importer: # object key(s) should already be populated continue - # elif field not in source_data: + # if field not in source_data: # # no source data for field # continue - elif field in fields: + if field in fields: # field is eligible for update generally, so compare # values between records @@ -1091,7 +1097,7 @@ class Importer: return obj - def can_delete_object(self, obj, data=None): + def can_delete_object(self, obj, data=None): # pylint: disable=unused-argument """ Should return true or false indicating whether the given object "can" be deleted. Default is to return true in all @@ -1110,7 +1116,7 @@ class Importer: """ return True - def delete_target_object(self, obj): + def delete_target_object(self, obj): # pylint: disable=unused-argument """ Delete the given raw object from the target side, and return true if successful. @@ -1174,6 +1180,8 @@ class FromFile(Importer): :meth:`close_input_file()`. """ + input_file = None + def setup(self): """ Open the input file. See also :meth:`open_input_file()`. @@ -1267,6 +1275,8 @@ class ToSqlalchemy(Importer): caches_target = True "" # nb. suppress sphinx docs + target_session = None + def get_target_object(self, key): """ Tries to fetch the object from target DB using ORM query. @@ -1282,7 +1292,7 @@ class ToSqlalchemy(Importer): try: return query.one() except orm.exc.NoResultFound: - pass + return None def get_target_objects(self, source_data=None, progress=None): """ @@ -1292,7 +1302,7 @@ class ToSqlalchemy(Importer): query = self.get_target_query(source_data=source_data) return query.all() - def get_target_query(self, source_data=None): + def get_target_query(self, source_data=None): # pylint: disable=unused-argument """ Returns an ORM query suitable to fetch existing objects from the target side. This is called from @@ -1300,7 +1310,7 @@ class ToSqlalchemy(Importer): """ return self.target_session.query(self.model_class) - def create_target_object(self, key, source_data): + def create_target_object(self, key, source_data): # pylint: disable=empty-docstring """ """ with self.target_session.no_autoflush: obj = super().create_target_object(key, source_data) @@ -1308,8 +1318,9 @@ class ToSqlalchemy(Importer): # nb. add new object to target db session self.target_session.add(obj) return obj + return None - def delete_target_object(self, obj): + def delete_target_object(self, obj): # pylint: disable=empty-docstring """ """ self.target_session.delete(obj) return True diff --git a/src/wuttasync/importing/csv.py b/src/wuttasync/importing/csv.py index a5db421..1d6946d 100644 --- a/src/wuttasync/importing/csv.py +++ b/src/wuttasync/importing/csv.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -42,7 +42,7 @@ from .model import ToWutta log = logging.getLogger(__name__) -class FromCsv(FromFile): +class FromCsv(FromFile): # pylint: disable=abstract-method """ Base class for importer/exporter using CSV file as data source. @@ -61,6 +61,8 @@ class FromCsv(FromFile): :class:`python:csv.DictReader` instance. """ + input_reader = None + csv_encoding = "utf_8" """ Encoding used by the CSV input file. @@ -104,7 +106,9 @@ class FromCsv(FromFile): """ path = self.get_input_file_path() log.debug("opening input file: %s", path) - self.input_file = open(path, "rt", encoding=self.csv_encoding) + self.input_file = open( # pylint: disable=consider-using-with + path, "rt", encoding=self.csv_encoding + ) self.input_reader = csv.DictReader(self.input_file) # nb. importer may have all supported fields by default, so @@ -118,7 +122,7 @@ class FromCsv(FromFile): self.input_file.close() raise ValueError("input file has no recognized fields") - def close_input_file(self): + def close_input_file(self): # pylint: disable=empty-docstring """ """ self.input_file.close() del self.input_reader @@ -136,7 +140,7 @@ class FromCsv(FromFile): return list(self.input_reader) -class FromCsvToSqlalchemyMixin: +class FromCsvToSqlalchemyMixin: # pylint: disable=too-few-public-methods """ Mixin class for CSV → SQLAlchemy ORM :term:`importers `. @@ -161,7 +165,7 @@ class FromCsvToSqlalchemyMixin: if isinstance(attr.prop.columns[0].type, UUID): self.uuid_keys.append(field) - def normalize_source_object(self, obj): + def normalize_source_object(self, obj): # pylint: disable=empty-docstring """ """ data = dict(obj) @@ -292,6 +296,6 @@ class FromCsvToWutta(FromCsvToSqlalchemyHandlerMixin, FromFileHandler, ToWuttaHa ToImporterBase = ToWutta - def get_target_model(self): + def get_target_model(self): # pylint: disable=empty-docstring """ """ return self.app.model diff --git a/src/wuttasync/importing/handlers.py b/src/wuttasync/importing/handlers.py index f9ba772..e9c6ac3 100644 --- a/src/wuttasync/importing/handlers.py +++ b/src/wuttasync/importing/handlers.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -275,7 +275,7 @@ class ImportHandler(GenericHandler): ) except: - # TODO: what should happen here? + log.exception("what should happen here?") # TODO raise else: @@ -497,7 +497,7 @@ class ImportHandler(GenericHandler): factory = self.importers[key] return factory(self.config, **kwargs) - def get_importer_kwargs(self, key, **kwargs): + def get_importer_kwargs(self, key, **kwargs): # pylint: disable=unused-argument """ Returns a dict of kwargs to be used when construcing an importer/exporter with the given key. This is normally called @@ -522,7 +522,7 @@ class FromFileHandler(ImportHandler): logic. """ - def process_data(self, *keys, **kwargs): + def process_data(self, *keys, **kwargs): # pylint: disable=empty-docstring """ """ # interpret file vs. folder path @@ -586,7 +586,7 @@ class ToSqlalchemyHandler(ImportHandler): """ raise NotImplementedError - def get_importer_kwargs(self, key, **kwargs): + def get_importer_kwargs(self, key, **kwargs): # pylint: disable=empty-docstring """ """ kwargs = super().get_importer_kwargs(key, **kwargs) kwargs.setdefault("target_session", self.target_session) diff --git a/src/wuttasync/importing/wutta.py b/src/wuttasync/importing/wutta.py index 18d4145..9de4822 100644 --- a/src/wuttasync/importing/wutta.py +++ b/src/wuttasync/importing/wutta.py @@ -2,7 +2,7 @@ ################################################################################ # # WuttaSync -- Wutta Framework for data import/export and real-time sync -# Copyright © 2024 Lance Edgar +# Copyright © 2024-2025 Lance Edgar # # This file is part of Wutta Framework. # @@ -36,7 +36,7 @@ class ToWuttaHandler(ToSqlalchemyHandler): target_key = "wutta" "" # nb. suppress docs - def get_target_title(self): + def get_target_title(self): # pylint: disable=empty-docstring """ """ # nb. we override parent to use app title as default if hasattr(self, "target_title"): diff --git a/tox.ini b/tox.ini index 78d41eb..a9472fb 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,10 @@ envlist = py38, py39, py310, py311 extras = tests commands = pytest {posargs} +[testenv:pylint] +basepython = python3.11 +commands = pylint wuttasync + [testenv:coverage] basepython = python3.11 commands = pytest --cov=wuttasync --cov-report=html --cov-fail-under=100