rattail/rattail
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Misc. changes for vendor catalog batch and related features

Browse source 
not very targeted, but all is solid i think..
This commit is contained in:
Lance Edgar 2022-01-07 15:01:37 -06:00
parent 0bec971b8b
commit 1ac0139fd3
16 changed files with 519 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
docs/api/index.rst
View file

@ -20,6 +20,7 @@ attributes and method signatures etc.
rattail/batch/pricing
rattail/batch/product
rattail/batch/purchase
rattail/batch/vendorcatalog
rattail/board
rattail/bouncer/index
rattail/clientele
@ -33,6 +34,7 @@ attributes and method signatures etc.
rattail/db/index
rattail/employment
rattail/enum
rattail/excel
rattail/exceptions
rattail/features/index
rattail/filemon/index
@ -62,4 +64,5 @@ attributes and method signatures etc.
rattail/trainwreck/index
rattail/upgrades
rattail/util
rattail/vendors.catalogs
rattail/win32

6
docs/api/rattail/batch/vendorcatalog.rst Normal file
View file

@ -0,0 +1,6 @@
``rattail.batch.vendorcatalog``
===============================
.. automodule:: rattail.batch.vendorcatalog
:members:

6
docs/api/rattail/excel.rst Normal file
View file

@ -0,0 +1,6 @@
``rattail.excel``
=================
.. automodule:: rattail.excel
:members:

6
docs/api/rattail/vendors.catalogs.rst Normal file
View file

@ -0,0 +1,6 @@
``rattail.vendors.catalogs``
============================
.. automodule:: rattail.vendors.catalogs
:members:

2
docs/conf.py
View file

@ -123,7 +123,7 @@ todo_include_todos = True
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'
html_theme = 'classic'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the

9
rattail/app.py
View file

@ -38,6 +38,7 @@ from rattail.util import (load_object, load_entry_points,
from rattail.files import temp_path
from rattail.mail import send_email
from rattail.config import parse_list
from rattail.core import get_uuid
class AppHandler(object):
@ -763,6 +764,14 @@ class AppHandler(object):
kwargs['dir'] = tmpdir
return temp_path(**kwargs)
def make_uuid(self):
"""
Generate a new UUID value.
:returns: UUID value as 32-character string.
"""
return get_uuid()
def normalize_phone_number(self, number, **kwargs):
"""
Normalize the given phone number, to a "common" format that

103
rattail/batch/vendorcatalog.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2021 Lance Edgar
# Copyright © 2010-2022 Lance Edgar
#
# This file is part of Rattail.
#
@ -26,6 +26,8 @@ Handler for Vendor Catalog batches
from __future__ import unicode_literals, absolute_import
import decimal
from sqlalchemy import orm
from rattail.db import model
@ -88,22 +90,43 @@ class VendorCatalogHandler(BatchHandler):
def populate_from_file(self, batch, progress=None):
"""
Pre-fill batch with row data from an input data file, leveraging a
specific catalog parser.
Populate the given batch using data from its input file. A
catalog parser will be instantiated and asked to read row data
from the file. Each row is then added to the batch.
The batch must have valid
:attr:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatch.filename`
and
:attr:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatch.parser_key`
attributes. The path to the input file will be determined by
invoking the
:meth:`~rattail.db.model.batch.core.BatchMixin.filepath()`
method on the batch.
:param batch: The batch to be populated.
:param progress: Optional progress factory.
"""
assert batch.filename and batch.parser_key
if not batch.filename:
raise ValueError("batch does not have a filename: {}".format(batch))
if not batch.parser_key:
raise ValueError("batch does not have a parser_key: {}".format(batch))
session = orm.object_session(batch)
path = batch.filepath(self.config)
parser = require_catalog_parser(batch.parser_key)
# TODO: should add `config` kwarg to CatalogParser constructor
parser.config = self.config
parser.app = self.app
parser.model = self.model
parser.enum = self.enum
parser.session = session
parser.vendor = batch.vendor
batch.effective = parser.parse_effective_date(path)
def append(row, i):
self.add_row(batch, row)
if i % 1000 == 0:
if i % 1000 == 0: # pragma: no cover
session.flush()
data = list(parser.parse_rows(path, progress=progress))
@ -111,35 +134,95 @@ class VendorCatalogHandler(BatchHandler):
message="Adding initial rows to batch")
def identify_product(self, row):
"""
Try to locate the product represented by the given row.
Lookups are done using either the ``upc`` or ``vendor_code``
attributes of the row.
Under normal circumstances the batch handler will have
pre-cached all existing products, for quicker lookup. For
instance this is the case for the full populate and refresh
actions. But this logic is able to do its own slower lookups
if there is no cache available.
:param row: A
:class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
instance.
:returns: A :class:`~rattail.db.model.products.Product`
instance, or ``None`` if no match could be found.
"""
products_handler = self.app.get_products_handler()
session = self.app.get_session(row)
product = None
if row.upc:
if hasattr(self, 'products'):
product = self.products['upc'].get(row.upc)
else:
product = products_handler.locate_product_for_gpc(
session, row.upc)
if not product and row.vendor_code:
if hasattr(self, 'products'):
product = self.products['vendor_code'].get(row.vendor_code)
else:
product = products_handler.locate_product_for_entry(
session, row.vendor_code, product_key='ignore',
lookup_vendor_code=True, vendor=row.batch.vendor)
return product
def refresh_row(self, row):
"""
Inspect a single row from a catalog, and set its attributes based on
whether or not the product exists, if we already have a cost record for
the vendor, if the catalog contains a change etc. Note that the
product lookup is done first by UPC and then by vendor item code.
Refresh data attributes and status for the given row.
For a vendor catalog, the typical thing is done for basic
product attributes.
If case cost is known but unit cost is not, the latter will be
calculated if possible.
"Old" (i.e. "current" prior to batch execution) values will
all be re-fetched from the main database(s), and "diff" values
will be re-calculated.
:param row: A
:class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
instance.
"""
batch = row.batch
# clear this first in case it's set
row.status_text = None
if not row.product:
row.product = self.identify_product(row)
if not row.product:
row.status_code = row.STATUS_PRODUCT_NOT_FOUND
return
product = row.product
row.upc = row.product.upc
row.item_id = row.product.item_id
row.brand_name = row.product.brand.name if row.product.brand else None
row.description = row.product.description
row.size = row.product.size
old_cost = row.product.vendor_cost
# maybe calculate unit cost from case cost
if row.unit_cost is None and row.case_cost is not None:
if row.case_size is not None:
# nb. sometimes both case size and cost are integers,
# and simple division yields a float! this approach
# should hopefully work regardless, to get a decimal.
row.unit_cost = decimal.Decimal('{:0.4f}'.format(
row.case_cost / row.case_size))
if hasattr(product, 'vendor_cost'):
old_cost = product.vendor_cost
else:
old_cost = product.cost_for_vendor(batch.vendor)
if not old_cost:
row.status_code = row.STATUS_NEW_COST
return

1
rattail/contrib/vendors/catalogs/generic.py vendored
View file

@ -62,6 +62,7 @@ class GenericCatalogParser(CatalogParser):
if not upc or not six.text_type(upc).strip():
continue # skip lines with no UPC value
upc = str(upc).replace(' ', '').replace('-', '')
row.item_entry = upc
row.upc = GPC(upc, calc_check_digit=False if len(upc) in (12, 13) else 'upc')
# cost values (required in some combination)

28
rattail/excel.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2021 Lance Edgar
# Copyright © 2010-2022 Lance Edgar
#
# This file is part of Rattail.
#
@ -111,10 +111,23 @@ class ExcelReaderXLSX(object):
"""
Basic class for reading Excel 2010 (.xslx) files.
Uses the ``openpyxl`` package to read the files.
Uses the `openpyxl`_ package to read the files.
.. _openpyxl: https://openpyxl.readthedocs.io/en/stable/
:param path: Path to the Excel data file.
:param header_row: 1-based row number which contains the header,
with field names.
:param strip_fieldnames: If true (the default), any whitespace
surrounding the field names will be stripped, i.e. after they
are read from the header row. Pass ``False`` here to suppress
the behavior and leave whitespace intact.
"""
def __init__(self, path, header_row=1, **kwargs):
def __init__(self, path, header_row=1, strip_fieldnames=True,
**kwargs):
"""
Constructor; opens an Excel file for reading.
@ -123,7 +136,11 @@ class ExcelReaderXLSX(object):
:param header_row: Which row contains the column headers. This is
1-based, so the 1 is the default (i.e. the first row).
"""
self.book = openpyxl.load_workbook(filename=path)
# nb. after much use with no problems, eventually did come
# across a spreadsheet which contained formula instead of
# values for certain cells. so now using ``data_only=True``
# to avoid the formula, hopefully nothing else breaks..
self.book = openpyxl.load_workbook(filename=path, data_only=True)
self.sheet = self.book.active
self.header_row = header_row
@ -137,6 +154,9 @@ class ExcelReaderXLSX(object):
assert self.fields is None
self.fields = list(row)
if strip_fieldnames:
self.fields = [field.strip() for field in self.fields]
def iter_rows(self):
return self.sheet.iter_rows(min_row=self.header_row + 1,
values_only=True)

22
rattail/products.py
View file

@ -2,7 +2,7 @@
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2021 Lance Edgar
# Copyright © 2010-2022 Lance Edgar
#
# This file is part of Rattail.
#
@ -47,12 +47,26 @@ class ProductsHandler(GenericHandler):
particular product can be deleted, etc.
"""
def make_gpc(self, value, **kwargs):
def make_gpc(self, value, ignore_errors=False, **kwargs):
"""
Try to convert the given value to a :class:`~rattail.gpc.GPC`
instance, and return the result.
Try to convert the given value to a GPC, and return the
result.
:param value: Value to be converted. This should be either a
string or integer value.
:param ignore_errors: If ``value`` is not valid for a GPC, an
error will be raised unless this param is set to true.
:returns: A :class:`~rattail.gpc.GPC` instance. Or, if the
``value`` is not valid, and ``ignore_errors`` was true,
then returns ``None``.
"""
try:
return GPC(value, **kwargs)
except:
if not ignore_errors:
raise
def make_full_description(self, product=None,
brand_name=None, description=None, size=None,

32
rattail/vendors/catalogs.py vendored
View file

@ -2,7 +2,7 @@
################################################################################
#
# Rattail -- Retail Software Framework
# Copyright © 2010-2018 Lance Edgar
# Copyright © 2010-2022 Lance Edgar
#
# This file is part of Rattail.
#
@ -38,7 +38,15 @@ class CatalogParser(object):
"""
Base class for all vendor catalog parsers.
.. attr:: vendor_key
.. note::
As of this writing the ``config`` param is technically optional
for the class constructor method, but that will certainly
change some day. Please be sure to pass a ``config`` param
when instantiating parsers in your code.
.. attribute:: vendor_key
Key for the vendor. This key will be used to locate an entry in the
settings table, e.g. ``'rattail.vendor.unfi'`` for a key of ``'unfi'``.
The value of this setting must be an exact match to either a
@ -49,7 +57,12 @@ class CatalogParser(object):
"""
vendor_key = None
# TODO: should add constructor, to accept `config` kwarg at least
def __init__(self, config=None, **kwargs):
if config:
self.config = config
self.app = config.get_app()
self.model = config.get_model()
self.enum = config.get_enum()
@property
def key(self):
@ -69,6 +82,19 @@ class CatalogParser(object):
"""
raise NotImplementedError("Catalog parser has no `parse_rows()` method: {0}".format(repr(self.key)))
def make_row(self):
"""
Create and return a new row, suitable for use in a vendor
catalog batch. The row will be empty and not yet part of any
database session.
:returns: A
:class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
instance.
"""
model = self.model
return model.VendorCatalogBatchRow()
def decimal(self, value, scale=4):
"""
Convert a value to a decimal, unless it's ``None``.

201
tests/batch/test_vendorcatalog.py Normal file
View file

@ -0,0 +1,201 @@
# -*- coding: utf-8; -*-
from __future__ import unicode_literals, absolute_import
import os
import shutil
import decimal
from unittest import TestCase
import sqlalchemy as sa
from rattail.batch import vendorcatalog as mod
from rattail.config import make_config
from rattail.db import Session
from rattail.excel import ExcelWriter
from rattail.gpc import GPC
class TestProductBatchHandler(TestCase):
def setUp(self):
self.config = self.make_config()
self.handler = self.make_handler()
def make_config(self):
return make_config([], extend=False)
def make_handler(self):
return mod.VendorCatalogHandler(self.config)
def test_populate_from_file(self):
engine = sa.create_engine('sqlite://')
model = self.config.get_model()
model.Base.metadata.create_all(bind=engine)
session = Session(bind=engine)
app = self.config.get_app()
# we'll need a user to create the batches
user = model.User(username='ralph')
session.add(user)
# make root folder to contain all temp files
tempdir = app.make_temp_dir()
# generate sample xlsx file
path = os.path.join(tempdir, 'sample.xlsx')
writer = ExcelWriter(path, ['UPC', 'Vendor Code', 'Unit Cost'])
writer.write_header()
writer.write_row(['074305001321', '123456', 4.19], row=2)
writer.save()
# make, configure folder for batch files
filesdir = os.path.join(tempdir, 'batch_files')
os.makedirs(filesdir)
self.config.setdefault('rattail', 'batch.files', filesdir)
# make the basic batch
batch = model.VendorCatalogBatch(uuid=app.make_uuid(),
id=1, created_by=user)
session.add(batch)
# batch must have certain attributes, else error
self.assertRaises(ValueError, self.handler.populate_from_file, batch)
self.handler.set_input_file(batch, path) # sets batch.filename
self.assertRaises(ValueError, self.handler.populate_from_file, batch)
batch.parser_key = 'rattail.contrib.generic'
# and finally, test our method proper
self.handler.setup_populate(batch)
self.handler.populate_from_file(batch)
self.assertEqual(len(batch.data_rows), 1)
row = batch.data_rows[0]
self.assertEqual(row.item_entry, '074305001321')
self.assertEqual(row.vendor_code, '123456')
self.assertEqual(row.unit_cost, decimal.Decimal('4.19'))
shutil.rmtree(tempdir)
session.rollback()
session.close()
def test_identify_product(self):
engine = sa.create_engine('sqlite://')
model = self.config.get_model()
model.Base.metadata.create_all(bind=engine)
session = Session(bind=engine)
app = self.config.get_app()
# make a test user, vendor, product, cost
user = model.User(username='ralph')
session.add(user)
vendor = model.Vendor()
session.add(vendor)
product = model.Product(upc=GPC('074305001321'))
session.add(product)
cost = model.ProductCost(vendor=vendor,
code='123456',
case_size=12,
case_cost=decimal.Decimal('54.00'),
unit_cost=decimal.Decimal('4.50'))
product.costs.append(cost)
# also a batch to contain the rows
batch = model.VendorCatalogBatch(uuid=app.make_uuid(),
id=1, created_by=user,
vendor=vendor,
filename='sample.xlsx',
parser_key='rattail.contrib.generic')
session.add(batch)
# row w/ no interesting attributes cannot yield a product
row = model.VendorCatalogBatchRow()
batch.data_rows.append(row)
result = self.handler.identify_product(row)
self.assertIsNone(result)
# but if we give row a upc, product is found
row.upc = GPC('074305001321')
result = self.handler.identify_product(row)
self.assertIs(result, product)
# now try one with vendor code instead of upc
row = model.VendorCatalogBatchRow(vendor_code='123456')
batch.data_rows.append(row)
result = self.handler.identify_product(row)
self.assertIs(result, product)
session.rollback()
session.close()
def test_refresh_row(self):
engine = sa.create_engine('sqlite://')
model = self.config.get_model()
model.Base.metadata.create_all(bind=engine)
session = Session(bind=engine)
app = self.config.get_app()
# make a test user, vendor, product
user = model.User(username='ralph')
session.add(user)
vendor = model.Vendor()
session.add(vendor)
product = model.Product(upc=GPC('074305001321'))
session.add(product)
# also a batch to contain the rows
batch = model.VendorCatalogBatch(uuid=app.make_uuid(),
id=1, created_by=user,
vendor=vendor,
filename='sample.xlsx',
parser_key='rattail.contrib.generic')
session.add(batch)
# empty row is just marked as product not found
row = model.VendorCatalogBatchRow()
batch.data_rows.append(row)
self.handler.refresh_row(row)
self.assertEqual(row.status_code, row.STATUS_PRODUCT_NOT_FOUND)
# row with upc is matched with product; also make sure unit
# cost is calculated from case cost
row = model.VendorCatalogBatchRow(upc=GPC('074305001321'),
case_size=12,
case_cost=decimal.Decimal('58.00'))
batch.data_rows.append(row)
self.handler.refresh_row(row)
self.assertIs(row.product, product)
self.assertEqual(row.status_code, row.STATUS_NEW_COST)
self.assertEqual(row.case_cost, 58)
self.assertEqual(row.case_size, 12)
self.assertEqual(row.unit_cost, decimal.Decimal('4.8333'))
# now we add a cost to the master product, and make sure new
# row will reflect an update for that cost
cost = model.ProductCost(vendor=vendor,
case_size=12,
case_cost=decimal.Decimal('54.00'),
unit_cost=decimal.Decimal('4.50'))
product.costs.append(cost)
row = model.VendorCatalogBatchRow(upc=GPC('074305001321'),
case_size=12,
case_cost=decimal.Decimal('58.00'))
batch.data_rows.append(row)
self.handler.refresh_row(row)
self.assertIs(row.product, product)
self.assertEqual(row.status_code, row.STATUS_CHANGE_COST)
self.assertEqual(row.old_case_cost, 54)
self.assertEqual(row.case_cost, 58)
self.assertEqual(row.old_unit_cost, decimal.Decimal('4.50'))
self.assertEqual(row.unit_cost, decimal.Decimal('4.8333'))
# and finally let's refresh everything, note that row #2
# should now *also* get "change cost" status
row = batch.data_rows[1]
self.assertEqual(row.status_code, row.STATUS_NEW_COST)
self.handler.setup_refresh(batch)
for row in batch.data_rows:
self.handler.refresh_row(row)
self.assertEqual(row.status_code, row.STATUS_CHANGE_COST)
session.rollback()
session.close()

40
tests/test_excel.py Normal file
View file

@ -0,0 +1,40 @@
# -*- coding: utf-8; -*-
from __future__ import unicode_literals, absolute_import
import os
from unittest import TestCase
import openpyxl
from rattail import excel as mod
from rattail.config import make_config
class TestExcelReaderXLSX(TestCase):
def setUp(self):
self.config = self.make_config()
def make_config(self):
return make_config([], extend=False)
def test_strip_fieldnames(self):
app = self.config.get_app()
path = app.make_temp_file(suffix='.xlsx')
# first make a workbook which has whitespace in column headers
workbook = openpyxl.Workbook()
sheet = workbook.active
sheet.append([' first ', 'second '])
workbook.save(path)
# reader should strip fieldnames by default
reader = mod.ExcelReaderXLSX(path)
self.assertEqual(reader.fields, ['first', 'second'])
# unless we say not to strip them
reader = mod.ExcelReaderXLSX(path, strip_fieldnames=False)
self.assertEqual(reader.fields, [' first ', 'second '])
os.remove(path)

43
tests/test_products.py Normal file
View file

@ -0,0 +1,43 @@
# -*- coding: utf-8; -*-
from __future__ import unicode_literals, absolute_import
from unittest import TestCase
import six
from rattail import products as mod
from rattail.config import make_config
from rattail.gpc import GPC
class TestProductsHandler(TestCase):
def setUp(self):
self.config = self.make_config()
self.handler = self.make_handler()
def make_config(self):
return make_config([], extend=False)
def make_handler(self):
return mod.ProductsHandler(self.config)
def test_make_gpc(self):
# basic real-world example
result = self.handler.make_gpc('74305001321')
self.assertIsInstance(result, GPC)
self.assertEqual(six.text_type(result), '00074305001321')
# and let it calculate check digit
result = self.handler.make_gpc('7430500132', calc_check_digit='upc')
self.assertIsInstance(result, GPC)
self.assertEqual(six.text_type(result), '00074305001321')
# bad one should raise error
self.assertRaises(ValueError, self.handler.make_gpc, 'BAD_VALUE')
# unless we suppress errors
result = self.handler.make_gpc('BAD_VALUE', ignore_errors=True)
self.assertIsNone(result)

0
tests/vendors/__init__.py vendored Normal file
View file

33
tests/vendors/test_catalogs.py vendored Normal file
View file

@ -0,0 +1,33 @@
# -*- coding: utf-8; -*-
from __future__ import unicode_literals, absolute_import
from unittest import TestCase
from rattail.vendors import catalogs as mod
from rattail.config import make_config
class TestCatalogParser(TestCase):
def setUp(self):
self.config = self.make_config()
self.parser = self.make_parser()
def make_config(self):
return make_config([], extend=False)
def make_parser(self):
return mod.CatalogParser(self.config)
def test_key_required(self):
# someone must define the parser key
self.assertRaises(NotImplementedError, getattr, self.parser, 'key')
def test_make_row(self):
model = self.config.get_model()
# make a basic row, it should work
row = self.parser.make_row()
self.assertIsInstance(row, model.VendorCatalogBatchRow)