Misc. changes for vendor catalog batch and related features

not very targeted, but all is solid i think..

docs/api/index.rst

@@ -20,6 +20,7 @@ attributes and method signatures etc.
    rattail/batch/pricing
    rattail/batch/product
    rattail/batch/purchase
+   rattail/batch/vendorcatalog
    rattail/board
    rattail/bouncer/index
    rattail/clientele
@@ -33,6 +34,7 @@ attributes and method signatures etc.
    rattail/db/index
    rattail/employment
    rattail/enum
+   rattail/excel
    rattail/exceptions
    rattail/features/index
    rattail/filemon/index
@@ -62,4 +64,5 @@ attributes and method signatures etc.
    rattail/trainwreck/index
    rattail/upgrades
    rattail/util
+   rattail/vendors.catalogs
    rattail/win32

docs/api/rattail/batch/vendorcatalog.rst

@@ -0,0 +1,6 @@
+
+``rattail.batch.vendorcatalog``
+===============================
+
+.. automodule:: rattail.batch.vendorcatalog
+   :members:

docs/api/rattail/excel.rst

@@ -0,0 +1,6 @@
+
+``rattail.excel``
+=================
+
+.. automodule:: rattail.excel
+   :members:

docs/api/rattail/vendors.catalogs.rst

@@ -0,0 +1,6 @@
+
+``rattail.vendors.catalogs``
+============================
+
+.. automodule:: rattail.vendors.catalogs
+   :members:

docs/conf.py

@@ -123,7 +123,7 @@ todo_include_todos = True
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'alabaster'
+html_theme = 'classic'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

rattail/app.py

@@ -38,6 +38,7 @@ from rattail.util import (load_object, load_entry_points,
 from rattail.files import temp_path
 from rattail.mail import send_email
 from rattail.config import parse_list
+from rattail.core import get_uuid
 
 
 class AppHandler(object):
@@ -763,6 +764,14 @@ class AppHandler(object):
                 kwargs['dir'] = tmpdir
         return temp_path(**kwargs)
 
+    def make_uuid(self):
+        """
+        Generate a new UUID value.
+
+        :returns: UUID value as 32-character string.
+        """
+        return get_uuid()
+
     def normalize_phone_number(self, number, **kwargs):
         """
         Normalize the given phone number, to a "common" format that

rattail/batch/vendorcatalog.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2021 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -26,6 +26,8 @@ Handler for Vendor Catalog batches
 
 from __future__ import unicode_literals, absolute_import
 
+import decimal
+
 from sqlalchemy import orm
 
 from rattail.db import model
@@ -88,22 +90,43 @@ class VendorCatalogHandler(BatchHandler):
 
     def populate_from_file(self, batch, progress=None):
         """
-        Pre-fill batch with row data from an input data file, leveraging a
+        Populate the given batch using data from its input file.  A
-        specific catalog parser.
+        catalog parser will be instantiated and asked to read row data
+        from the file.  Each row is then added to the batch.
+
+        The batch must have valid
+        :attr:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatch.filename`
+        and
+        :attr:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatch.parser_key`
+        attributes.  The path to the input file will be determined by
+        invoking the
+        :meth:`~rattail.db.model.batch.core.BatchMixin.filepath()`
+        method on the batch.
+
+        :param batch: The batch to be populated.
+
+        :param progress: Optional progress factory.
         """
-        assert batch.filename and batch.parser_key
+        if not batch.filename:
+            raise ValueError("batch does not have a filename: {}".format(batch))
+        if not batch.parser_key:
+            raise ValueError("batch does not have a parser_key: {}".format(batch))
+
         session = orm.object_session(batch)
         path = batch.filepath(self.config)
         parser = require_catalog_parser(batch.parser_key)
         # TODO: should add `config` kwarg to CatalogParser constructor
         parser.config = self.config
+        parser.app = self.app
+        parser.model = self.model
+        parser.enum = self.enum
         parser.session = session
         parser.vendor = batch.vendor
         batch.effective = parser.parse_effective_date(path)
 
         def append(row, i):
             self.add_row(batch, row)
-            if i % 1000 == 0:
+            if i % 1000 == 0: # pragma: no cover
                 session.flush()
 
         data = list(parser.parse_rows(path, progress=progress))
@@ -111,35 +134,95 @@ class VendorCatalogHandler(BatchHandler):
                            message="Adding initial rows to batch")
 
     def identify_product(self, row):
+        """
+        Try to locate the product represented by the given row.
+        Lookups are done using either the ``upc`` or ``vendor_code``
+        attributes of the row.
+
+        Under normal circumstances the batch handler will have
+        pre-cached all existing products, for quicker lookup.  For
+        instance this is the case for the full populate and refresh
+        actions.  But this logic is able to do its own slower lookups
+        if there is no cache available.
+
+        :param row: A
+           :class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
+           instance.
+
+        :returns: A :class:`~rattail.db.model.products.Product`
+           instance, or ``None`` if no match could be found.
+        """
+        products_handler = self.app.get_products_handler()
+        session = self.app.get_session(row)
         product = None
+
         if row.upc:
+            if hasattr(self, 'products'):
                 product = self.products['upc'].get(row.upc)
+            else:
+                product = products_handler.locate_product_for_gpc(
+                    session, row.upc)
+
         if not product and row.vendor_code:
+            if hasattr(self, 'products'):
                 product = self.products['vendor_code'].get(row.vendor_code)
+            else:
+                product = products_handler.locate_product_for_entry(
+                    session, row.vendor_code, product_key='ignore',
+                    lookup_vendor_code=True, vendor=row.batch.vendor)
+
         return product
 
     def refresh_row(self, row):
         """
-        Inspect a single row from a catalog, and set its attributes based on
+        Refresh data attributes and status for the given row.
-        whether or not the product exists, if we already have a cost record for
+
-        the vendor, if the catalog contains a change etc.  Note that the
+        For a vendor catalog, the typical thing is done for basic
-        product lookup is done first by UPC and then by vendor item code.
+        product attributes.
+
+        If case cost is known but unit cost is not, the latter will be
+        calculated if possible.
+
+        "Old" (i.e. "current" prior to batch execution) values will
+        all be re-fetched from the main database(s), and "diff" values
+        will be re-calculated.
+
+        :param row: A
+           :class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
+           instance.
         """
+        batch = row.batch
+
         # clear this first in case it's set
         row.status_text = None
 
+        if not row.product:
             row.product = self.identify_product(row)
             if not row.product:
                 row.status_code = row.STATUS_PRODUCT_NOT_FOUND
                 return
 
+        product = row.product
+
         row.upc = row.product.upc
         row.item_id = row.product.item_id
         row.brand_name = row.product.brand.name if row.product.brand else None
         row.description = row.product.description
         row.size = row.product.size
 
-        old_cost = row.product.vendor_cost
+        # maybe calculate unit cost from case cost
+        if row.unit_cost is None and row.case_cost is not None:
+            if row.case_size is not None:
+                # nb. sometimes both case size and cost are integers,
+                # and simple division yields a float!  this approach
+                # should hopefully work regardless, to get a decimal.
+                row.unit_cost = decimal.Decimal('{:0.4f}'.format(
+                    row.case_cost / row.case_size))
+
+        if hasattr(product, 'vendor_cost'):
+            old_cost = product.vendor_cost
+        else:
+            old_cost = product.cost_for_vendor(batch.vendor)
         if not old_cost:
             row.status_code = row.STATUS_NEW_COST
             return

rattail/contrib/vendors/catalogs/generic.py

@@ -62,6 +62,7 @@ class GenericCatalogParser(CatalogParser):
             if not upc or not six.text_type(upc).strip():
                 continue        # skip lines with no UPC value
             upc = str(upc).replace(' ', '').replace('-', '')
+            row.item_entry = upc
             row.upc = GPC(upc, calc_check_digit=False if len(upc) in (12, 13) else 'upc')
 
             # cost values (required in some combination)

rattail/excel.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2021 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -111,10 +111,23 @@ class ExcelReaderXLSX(object):
     """
     Basic class for reading Excel 2010 (.xslx) files.
 
-    Uses the ``openpyxl`` package to read the files.
+    Uses the `openpyxl`_ package to read the files.
+
+    .. _openpyxl: https://openpyxl.readthedocs.io/en/stable/
+
+    :param path: Path to the Excel data file.
+
+    :param header_row: 1-based row number which contains the header,
+       with field names.
+
+    :param strip_fieldnames: If true (the default), any whitespace
+       surrounding the field names will be stripped, i.e. after they
+       are read from the header row.  Pass ``False`` here to suppress
+       the behavior and leave whitespace intact.
     """
 
-    def __init__(self, path, header_row=1, **kwargs):
+    def __init__(self, path, header_row=1, strip_fieldnames=True,
+                 **kwargs):
         """
         Constructor; opens an Excel file for reading.
 
@@ -123,7 +136,11 @@ class ExcelReaderXLSX(object):
         :param header_row: Which row contains the column headers.  This is
            1-based, so the 1 is the default (i.e. the first row).
         """
-        self.book = openpyxl.load_workbook(filename=path)
+        # nb. after much use with no problems, eventually did come
+        # across a spreadsheet which contained formula instead of
+        # values for certain cells.  so now using ``data_only=True``
+        # to avoid the formula, hopefully nothing else breaks..
+        self.book = openpyxl.load_workbook(filename=path, data_only=True)
         self.sheet = self.book.active
 
         self.header_row = header_row
@@ -137,6 +154,9 @@ class ExcelReaderXLSX(object):
             assert self.fields is None
             self.fields = list(row)
 
+        if strip_fieldnames:
+            self.fields = [field.strip() for field in self.fields]
+
     def iter_rows(self):
         return self.sheet.iter_rows(min_row=self.header_row + 1,
                                     values_only=True)

rattail/products.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2021 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -47,12 +47,26 @@ class ProductsHandler(GenericHandler):
     particular product can be deleted, etc.
     """
 
-    def make_gpc(self, value, **kwargs):
+    def make_gpc(self, value, ignore_errors=False, **kwargs):
         """
-        Try to convert the given value to a :class:`~rattail.gpc.GPC`
+        Try to convert the given value to a GPC, and return the
-        instance, and return the result.
+        result.
+
+        :param value: Value to be converted.  This should be either a
+           string or integer value.
+
+        :param ignore_errors: If ``value`` is not valid for a GPC, an
+           error will be raised unless this param is set to true.
+
+        :returns: A :class:`~rattail.gpc.GPC` instance.  Or, if the
+           ``value`` is not valid, and ``ignore_errors`` was true,
+           then returns ``None``.
         """
+        try:
             return GPC(value, **kwargs)
+        except:
+            if not ignore_errors:
+                raise
 
     def make_full_description(self, product=None,
                               brand_name=None, description=None, size=None,

rattail/vendors/catalogs.py

@@ -2,7 +2,7 @@
 ################################################################################
 #
 #  Rattail -- Retail Software Framework
-#  Copyright © 2010-2018 Lance Edgar
+#  Copyright © 2010-2022 Lance Edgar
 #
 #  This file is part of Rattail.
 #
@@ -38,7 +38,15 @@ class CatalogParser(object):
     """
     Base class for all vendor catalog parsers.
 
-    .. attr:: vendor_key
+    .. note::
+
+       As of this writing the ``config`` param is technically optional
+       for the class constructor method, but that will certainly
+       change some day.  Please be sure to pass a ``config`` param
+       when instantiating parsers in your code.
+
+    .. attribute:: vendor_key
+
        Key for the vendor.  This key will be used to locate an entry in the
        settings table, e.g. ``'rattail.vendor.unfi'`` for a key of ``'unfi'``.
        The value of this setting must be an exact match to either a
@@ -49,7 +57,12 @@ class CatalogParser(object):
     """
     vendor_key = None
 
-    # TODO: should add constructor, to accept `config` kwarg at least
+    def __init__(self, config=None, **kwargs):
+        if config:
+            self.config = config
+            self.app = config.get_app()
+            self.model = config.get_model()
+            self.enum = config.get_enum()
 
     @property
     def key(self):
@@ -69,6 +82,19 @@ class CatalogParser(object):
         """
         raise NotImplementedError("Catalog parser has no `parse_rows()` method: {0}".format(repr(self.key)))
 
+    def make_row(self):
+        """
+        Create and return a new row, suitable for use in a vendor
+        catalog batch.  The row will be empty and not yet part of any
+        database session.
+
+        :returns: A
+           :class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
+           instance.
+        """
+        model = self.model
+        return model.VendorCatalogBatchRow()
+
     def decimal(self, value, scale=4):
         """
         Convert a value to a decimal, unless it's ``None``.

tests/batch/test_vendorcatalog.py

@@ -0,0 +1,201 @@
+# -*- coding: utf-8; -*-
+
+from __future__ import unicode_literals, absolute_import
+
+import os
+import shutil
+import decimal
+from unittest import TestCase
+
+import sqlalchemy as sa
+
+from rattail.batch import vendorcatalog as mod
+from rattail.config import make_config
+from rattail.db import Session
+from rattail.excel import ExcelWriter
+from rattail.gpc import GPC
+
+
+class TestProductBatchHandler(TestCase):
+
+    def setUp(self):
+        self.config = self.make_config()
+        self.handler = self.make_handler()
+
+    def make_config(self):
+        return make_config([], extend=False)
+
+    def make_handler(self):
+        return mod.VendorCatalogHandler(self.config)
+
+    def test_populate_from_file(self):
+        engine = sa.create_engine('sqlite://')
+        model = self.config.get_model()
+        model.Base.metadata.create_all(bind=engine)
+        session = Session(bind=engine)
+        app = self.config.get_app()
+
+        # we'll need a user to create the batches
+        user = model.User(username='ralph')
+        session.add(user)
+
+        # make root folder to contain all temp files
+        tempdir = app.make_temp_dir()
+
+        # generate sample xlsx file
+        path = os.path.join(tempdir, 'sample.xlsx')
+        writer = ExcelWriter(path, ['UPC', 'Vendor Code', 'Unit Cost'])
+        writer.write_header()
+        writer.write_row(['074305001321', '123456', 4.19], row=2)
+        writer.save()
+
+        # make, configure folder for batch files
+        filesdir = os.path.join(tempdir, 'batch_files')
+        os.makedirs(filesdir)
+        self.config.setdefault('rattail', 'batch.files', filesdir)
+
+        # make the basic batch
+        batch = model.VendorCatalogBatch(uuid=app.make_uuid(),
+                                         id=1, created_by=user)
+        session.add(batch)
+
+        # batch must have certain attributes, else error
+        self.assertRaises(ValueError, self.handler.populate_from_file, batch)
+        self.handler.set_input_file(batch, path) # sets batch.filename
+        self.assertRaises(ValueError, self.handler.populate_from_file, batch)
+        batch.parser_key = 'rattail.contrib.generic'
+
+        # and finally, test our method proper
+        self.handler.setup_populate(batch)
+        self.handler.populate_from_file(batch)
+        self.assertEqual(len(batch.data_rows), 1)
+        row = batch.data_rows[0]
+        self.assertEqual(row.item_entry, '074305001321')
+        self.assertEqual(row.vendor_code, '123456')
+        self.assertEqual(row.unit_cost, decimal.Decimal('4.19'))
+
+        shutil.rmtree(tempdir)
+        session.rollback()
+        session.close()
+
+    def test_identify_product(self):
+        engine = sa.create_engine('sqlite://')
+        model = self.config.get_model()
+        model.Base.metadata.create_all(bind=engine)
+        session = Session(bind=engine)
+        app = self.config.get_app()
+
+        # make a test user, vendor, product, cost
+        user = model.User(username='ralph')
+        session.add(user)
+        vendor = model.Vendor()
+        session.add(vendor)
+        product = model.Product(upc=GPC('074305001321'))
+        session.add(product)
+        cost = model.ProductCost(vendor=vendor,
+                                 code='123456',
+                                 case_size=12,
+                                 case_cost=decimal.Decimal('54.00'),
+                                 unit_cost=decimal.Decimal('4.50'))
+        product.costs.append(cost)
+
+        # also a batch to contain the rows
+        batch = model.VendorCatalogBatch(uuid=app.make_uuid(),
+                                         id=1, created_by=user,
+                                         vendor=vendor,
+                                         filename='sample.xlsx',
+                                         parser_key='rattail.contrib.generic')
+        session.add(batch)
+
+        # row w/ no interesting attributes cannot yield a product
+        row = model.VendorCatalogBatchRow()
+        batch.data_rows.append(row)
+        result = self.handler.identify_product(row)
+        self.assertIsNone(result)
+
+        # but if we give row a upc, product is found
+        row.upc = GPC('074305001321')
+        result = self.handler.identify_product(row)
+        self.assertIs(result, product)
+
+        # now try one with vendor code instead of upc
+        row = model.VendorCatalogBatchRow(vendor_code='123456')
+        batch.data_rows.append(row)
+        result = self.handler.identify_product(row)
+        self.assertIs(result, product)
+
+        session.rollback()
+        session.close()
+
+    def test_refresh_row(self):
+        engine = sa.create_engine('sqlite://')
+        model = self.config.get_model()
+        model.Base.metadata.create_all(bind=engine)
+        session = Session(bind=engine)
+        app = self.config.get_app()
+
+        # make a test user, vendor, product
+        user = model.User(username='ralph')
+        session.add(user)
+        vendor = model.Vendor()
+        session.add(vendor)
+        product = model.Product(upc=GPC('074305001321'))
+        session.add(product)
+
+        # also a batch to contain the rows
+        batch = model.VendorCatalogBatch(uuid=app.make_uuid(),
+                                         id=1, created_by=user,
+                                         vendor=vendor,
+                                         filename='sample.xlsx',
+                                         parser_key='rattail.contrib.generic')
+        session.add(batch)
+
+        # empty row is just marked as product not found
+        row = model.VendorCatalogBatchRow()
+        batch.data_rows.append(row)
+        self.handler.refresh_row(row)
+        self.assertEqual(row.status_code, row.STATUS_PRODUCT_NOT_FOUND)
+
+        # row with upc is matched with product; also make sure unit
+        # cost is calculated from case cost
+        row = model.VendorCatalogBatchRow(upc=GPC('074305001321'),
+                                          case_size=12,
+                                          case_cost=decimal.Decimal('58.00'))
+        batch.data_rows.append(row)
+        self.handler.refresh_row(row)
+        self.assertIs(row.product, product)
+        self.assertEqual(row.status_code, row.STATUS_NEW_COST)
+        self.assertEqual(row.case_cost, 58)
+        self.assertEqual(row.case_size, 12)
+        self.assertEqual(row.unit_cost, decimal.Decimal('4.8333'))
+
+        # now we add a cost to the master product, and make sure new
+        # row will reflect an update for that cost
+        cost = model.ProductCost(vendor=vendor, 
+                                 case_size=12,
+                                 case_cost=decimal.Decimal('54.00'),
+                                 unit_cost=decimal.Decimal('4.50'))
+        product.costs.append(cost)
+        row = model.VendorCatalogBatchRow(upc=GPC('074305001321'),
+                                          case_size=12,
+                                          case_cost=decimal.Decimal('58.00'))
+        batch.data_rows.append(row)
+        self.handler.refresh_row(row)
+        self.assertIs(row.product, product)
+        self.assertEqual(row.status_code, row.STATUS_CHANGE_COST)
+        self.assertEqual(row.old_case_cost, 54)
+        self.assertEqual(row.case_cost, 58)
+        self.assertEqual(row.old_unit_cost, decimal.Decimal('4.50'))
+        self.assertEqual(row.unit_cost, decimal.Decimal('4.8333'))
+
+        # and finally let's refresh everything, note that row #2
+        # should now *also* get "change cost" status
+        row = batch.data_rows[1]
+        self.assertEqual(row.status_code, row.STATUS_NEW_COST)
+        self.handler.setup_refresh(batch)
+        for row in batch.data_rows:
+            self.handler.refresh_row(row)
+        self.assertEqual(row.status_code, row.STATUS_CHANGE_COST)
+
+        session.rollback()
+        session.close()

tests/test_excel.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8; -*-
+
+from __future__ import unicode_literals, absolute_import
+
+import os
+from unittest import TestCase
+
+import openpyxl
+
+from rattail import excel as mod
+from rattail.config import make_config
+
+
+class TestExcelReaderXLSX(TestCase):
+
+    def setUp(self):
+        self.config = self.make_config()
+
+    def make_config(self):
+        return make_config([], extend=False)
+
+    def test_strip_fieldnames(self):
+        app = self.config.get_app()
+        path = app.make_temp_file(suffix='.xlsx')
+
+        # first make a workbook which has whitespace in column headers
+        workbook = openpyxl.Workbook()
+        sheet = workbook.active
+        sheet.append([' first ', 'second   '])
+        workbook.save(path)
+
+        # reader should strip fieldnames by default
+        reader = mod.ExcelReaderXLSX(path)
+        self.assertEqual(reader.fields, ['first', 'second'])
+
+        # unless we say not to strip them
+        reader = mod.ExcelReaderXLSX(path, strip_fieldnames=False)
+        self.assertEqual(reader.fields, [' first ', 'second   '])
+
+        os.remove(path)

tests/test_products.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8; -*-
+
+from __future__ import unicode_literals, absolute_import
+
+from unittest import TestCase
+
+import six
+
+from rattail import products as mod
+from rattail.config import make_config
+from rattail.gpc import GPC
+
+
+class TestProductsHandler(TestCase):
+
+    def setUp(self):
+        self.config = self.make_config()
+        self.handler = self.make_handler()
+
+    def make_config(self):
+        return make_config([], extend=False)
+
+    def make_handler(self):
+        return mod.ProductsHandler(self.config)
+
+    def test_make_gpc(self):
+
+        # basic real-world example
+        result = self.handler.make_gpc('74305001321')
+        self.assertIsInstance(result, GPC)
+        self.assertEqual(six.text_type(result), '00074305001321')
+
+        # and let it calculate check digit
+        result = self.handler.make_gpc('7430500132', calc_check_digit='upc')
+        self.assertIsInstance(result, GPC)
+        self.assertEqual(six.text_type(result), '00074305001321')
+
+        # bad one should raise error
+        self.assertRaises(ValueError, self.handler.make_gpc, 'BAD_VALUE')
+
+        # unless we suppress errors
+        result = self.handler.make_gpc('BAD_VALUE', ignore_errors=True)
+        self.assertIsNone(result)

tests/vendors/test_catalogs.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8; -*-
+
+from __future__ import unicode_literals, absolute_import
+
+from unittest import TestCase
+
+from rattail.vendors import catalogs as mod
+from rattail.config import make_config
+
+
+class TestCatalogParser(TestCase):
+
+    def setUp(self):
+        self.config = self.make_config()
+        self.parser = self.make_parser()
+
+    def make_config(self):
+        return make_config([], extend=False)
+
+    def make_parser(self):
+        return mod.CatalogParser(self.config)
+
+    def test_key_required(self):
+
+        # someone must define the parser key
+        self.assertRaises(NotImplementedError, getattr, self.parser, 'key')
+
+    def test_make_row(self):
+        model = self.config.get_model()
+
+        # make a basic row, it should work
+        row = self.parser.make_row()
+        self.assertIsInstance(row, model.VendorCatalogBatchRow)