From af1e38aa18cc9096ecf97c7a88af7ef2f686f889 Mon Sep 17 00:00:00 2001 From: Lance Edgar Date: Wed, 4 Mar 2020 19:05:55 -0600 Subject: [PATCH] Oerhaul the Vendor import/export between Rattail and CORE also, add new DB schema specific to this integration, to hold PKs etc. --- rattail_corepos/config.py | 8 +-- rattail_corepos/corepos/importing/model.py | 11 +-- rattail_corepos/corepos/importing/rattail.py | 50 ++++++++++++-- rattail_corepos/corepos/util.py | 41 +++++++++++ rattail_corepos/datasync/corepos.py | 6 +- ...275_initial_core_pos_integration_tables.py | 51 ++++++++++++++ rattail_corepos/db/model.py | 68 +++++++++++++++++++ rattail_corepos/importing/__init__.py | 27 ++++++++ rattail_corepos/importing/corepos.py | 40 +++++++---- rattail_corepos/importing/model.py | 39 +++++++++++ 10 files changed, 311 insertions(+), 30 deletions(-) create mode 100644 rattail_corepos/corepos/util.py create mode 100644 rattail_corepos/db/alembic/versions/b43e93d32275_initial_core_pos_integration_tables.py create mode 100644 rattail_corepos/db/model.py create mode 100644 rattail_corepos/importing/model.py diff --git a/rattail_corepos/config.py b/rattail_corepos/config.py index 9be588e..69fe00f 100644 --- a/rattail_corepos/config.py +++ b/rattail_corepos/config.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2018 Lance Edgar +# Copyright © 2010-2020 Lance Edgar # # This file is part of Rattail. # @@ -24,8 +24,6 @@ Rattail-COREPOS Config Extension """ -from __future__ import unicode_literals, absolute_import - from rattail.config import ConfigExtension from rattail.db.config import get_engines @@ -37,8 +35,8 @@ class RattailCOREPOSExtension(ConfigExtension): key = 'rattail-corepos' def configure(self, config): - from corepos.db import Session as CoreSession - from corepos.trans.db import Session as CoreTransSession + from corepos.db.office_op import Session as CoreSession + from corepos.db.office_trans import Session as CoreTransSession engines = get_engines(config, section='corepos.db.office_op') config.corepos_engines = engines diff --git a/rattail_corepos/corepos/importing/model.py b/rattail_corepos/corepos/importing/model.py index 03afffa..84ab883 100644 --- a/rattail_corepos/corepos/importing/model.py +++ b/rattail_corepos/corepos/importing/model.py @@ -96,8 +96,7 @@ class VendorImporter(ToCoreAPI): 'halfCases', ] # TODO: this importer is in a bit of an experimental state at the moment. - # we only allow "update" b/c it will use the API instead of direct DB - allow_create = False + # we only allow create/update b/c it will use the API instead of direct DB allow_delete = False def get_local_objects(self, host_data=None): @@ -122,6 +121,10 @@ class VendorImporter(ToCoreAPI): return data + def create_object(self, key, data): + # we can get away with using the same logic for both here + return self.update_object(None, data) + def update_object(self, vendor, data, local_data=None): """ Push an update for the vendor, via the CORE API. @@ -130,5 +133,5 @@ class VendorImporter(ToCoreAPI): return data vendorID = data.pop('vendorID') - self.api.set_vendor(vendorID, **data) - return data + vendor = self.api.set_vendor(vendorID, **data) + return vendor diff --git a/rattail_corepos/corepos/importing/rattail.py b/rattail_corepos/corepos/importing/rattail.py index ae17551..cc8e253 100644 --- a/rattail_corepos/corepos/importing/rattail.py +++ b/rattail_corepos/corepos/importing/rattail.py @@ -28,8 +28,10 @@ import logging from rattail import importing from rattail.db import model +from rattail.db.util import short_session from rattail.util import OrderedDict from rattail_corepos.corepos import importing as corepos_importing +from rattail_corepos.corepos.util import get_max_existing_vendor_id log = logging.getLogger(__name__) @@ -71,17 +73,31 @@ class VendorImporter(FromRattail, corepos_importing.model.VendorImporter): 'email', ] + def setup(self): + super(VendorImporter, self).setup() + + # self.max_existing_vendor_id = self.get_max_existing_vendor_id() + self.max_existing_vendor_id = get_max_existing_vendor_id() + self.last_vendor_id = self.max_existing_vendor_id + + def get_next_vendor_id(self): + if hasattr(self, 'last_vendor_id'): + self.last_vendor_id += 1 + return self.last_vendor_id + + last_vendor_id = get_max_existing_vendor_id() + return last_vendor_id + 1 + def normalize_host_object(self, vendor): - if not vendor.id or not vendor.id.isdigit(): - log.warning("vendor %s has incompatible ID value: %s", - vendor.uuid, vendor.id) - return + vendor_id = vendor.corepos_id + if not vendor_id: + vendor_id = self.get_next_vendor_id() data = { - 'vendorID': vendor.id, + 'vendorID': str(vendor_id), 'vendorName': vendor.name, - 'vendorAbbreviation': vendor.abbreviation, - 'discountRate': float(vendor.special_discount), + 'vendorAbbreviation': vendor.abbreviation or '', + 'discountRate': float(vendor.special_discount or 0), } if 'phone' in self.fields: @@ -98,4 +114,24 @@ class VendorImporter(FromRattail, corepos_importing.model.VendorImporter): email = vendor.email data['email'] = email.address if email else '' + # also embed original Rattail vendor object, if we'll be needing to + # update it later with a new CORE ID + if not vendor.corepos_id: + data['_rattail_vendor'] = vendor + return data + + def create_object(self, key, data): + + # grab vendor object we (maybe) stashed when normalizing + rattail_vendor = data.pop('_rattail_vendor', None) + + # do normal create logic + vendor = super(VendorImporter, self).create_object(key, data) + if vendor: + + # maybe set the CORE ID for vendor in Rattail + if rattail_vendor: + rattail_vendor.corepos_id = int(vendor['vendorID']) + + return vendor diff --git a/rattail_corepos/corepos/util.py b/rattail_corepos/corepos/util.py new file mode 100644 index 0000000..cd5e34e --- /dev/null +++ b/rattail_corepos/corepos/util.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2020 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +CORE-POS misc. utilities +""" + +import sqlalchemy as sa + +from corepos.db.office_op import Session as CoreSession, model as corepos + +from rattail.db.util import short_session + + +def get_max_existing_vendor_id(session=None): + """ + Returns the "last" (max) existing value for the ``vendors.vendorID`` + column, for use when creating new records, since it is not auto-increment. + """ + with short_session(Session=CoreSession, session=session) as s: + return s.query(sa.func.max(corepos.Vendor.id))\ + .scalar() or 0 diff --git a/rattail_corepos/datasync/corepos.py b/rattail_corepos/datasync/corepos.py index 05056b2..8c8423f 100644 --- a/rattail_corepos/datasync/corepos.py +++ b/rattail_corepos/datasync/corepos.py @@ -160,8 +160,10 @@ class FromRattailToCore(NewDataSyncImportConsumer): ] for change in [c for c in changes if c.payload_type in types]: if change.payload_type == 'Vendor' and change.deletion: - # just do default logic for this one - self.invoke_importer(session, change) + # # just do default logic for this one + # self.invoke_importer(session, change) + # TODO: we have no way to delete a CORE vendor via API, right? + pass else: # we consider this a "vendor add/update" vendor = self.get_host_vendor(session, change) if vendor: diff --git a/rattail_corepos/db/alembic/versions/b43e93d32275_initial_core_pos_integration_tables.py b/rattail_corepos/db/alembic/versions/b43e93d32275_initial_core_pos_integration_tables.py new file mode 100644 index 0000000..e76c708 --- /dev/null +++ b/rattail_corepos/db/alembic/versions/b43e93d32275_initial_core_pos_integration_tables.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +"""initial CORE-POS integration tables + +Revision ID: b43e93d32275 +Revises: dfc1ed686f3f +Create Date: 2020-03-04 14:21:23.625568 + +""" + +# revision identifiers, used by Alembic. +revision = 'b43e93d32275' +down_revision = None +branch_labels = ('rattail_corepos',) +depends_on = None + +from alembic import op +import sqlalchemy as sa +import rattail.db.types + + + +def upgrade(): + + # corepos_vendor + op.create_table('corepos_vendor', + sa.Column('uuid', sa.String(length=32), nullable=False), + sa.Column('corepos_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['uuid'], ['vendor.uuid'], name='corepos_vendor_fk_vendor'), + sa.PrimaryKeyConstraint('uuid') + ) + op.create_table('corepos_vendor_version', + sa.Column('uuid', sa.String(length=32), autoincrement=False, nullable=False), + sa.Column('corepos_id', sa.Integer(), autoincrement=False, nullable=True), + sa.Column('transaction_id', sa.BigInteger(), autoincrement=False, nullable=False), + sa.Column('end_transaction_id', sa.BigInteger(), nullable=True), + sa.Column('operation_type', sa.SmallInteger(), nullable=False), + sa.PrimaryKeyConstraint('uuid', 'transaction_id') + ) + op.create_index(op.f('ix_corepos_vendor_version_end_transaction_id'), 'corepos_vendor_version', ['end_transaction_id'], unique=False) + op.create_index(op.f('ix_corepos_vendor_version_operation_type'), 'corepos_vendor_version', ['operation_type'], unique=False) + op.create_index(op.f('ix_corepos_vendor_version_transaction_id'), 'corepos_vendor_version', ['transaction_id'], unique=False) + + +def downgrade(): + + # corepos_vendor + op.drop_index(op.f('ix_corepos_vendor_version_transaction_id'), table_name='corepos_vendor_version') + op.drop_index(op.f('ix_corepos_vendor_version_operation_type'), table_name='corepos_vendor_version') + op.drop_index(op.f('ix_corepos_vendor_version_end_transaction_id'), table_name='corepos_vendor_version') + op.drop_table('corepos_vendor_version') + op.drop_table('corepos_vendor') diff --git a/rattail_corepos/db/model.py b/rattail_corepos/db/model.py new file mode 100644 index 0000000..6f4f87e --- /dev/null +++ b/rattail_corepos/db/model.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2020 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Database schema extensions for CORE-POS integration +""" + +import sqlalchemy as sa +from sqlalchemy import orm + +from rattail.db import model + + +__all__ = ['CoreVendor'] + + +class CoreVendor(model.Base): + """ + CORE-specific extensions to :class:`rattail:rattail.db.model.Vendor`. + """ + __tablename__ = 'corepos_vendor' + __table_args__ = ( + sa.ForeignKeyConstraint(['uuid'], ['vendor.uuid'], + name='corepos_vendor_fk_vendor'), + ) + __versioned__ = {} + + uuid = model.uuid_column(default=None) + vendor = orm.relationship( + model.Vendor, + doc=""" + Reference to the actual vendor record, which this one extends. + """, + backref=orm.backref( + '_corepos', + uselist=False, + cascade='all, delete-orphan', + doc=""" + Reference to the CORE-POS extension record for this vendor. + """)) + + corepos_id = sa.Column(sa.Integer(), nullable=False, doc=""" + ``vendorID`` value for the vendor, within CORE-POS. + """) + + def __str__(self): + return str(self.vendor) + +CoreVendor.make_proxy(model.Vendor, '_corepos', 'corepos_id') diff --git a/rattail_corepos/importing/__init__.py b/rattail_corepos/importing/__init__.py index e69de29..1947f94 100644 --- a/rattail_corepos/importing/__init__.py +++ b/rattail_corepos/importing/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2020 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Importing data into Rattail +""" + +from . import model diff --git a/rattail_corepos/importing/corepos.py b/rattail_corepos/importing/corepos.py index 91b45f0..1dd3e87 100644 --- a/rattail_corepos/importing/corepos.py +++ b/rattail_corepos/importing/corepos.py @@ -2,7 +2,7 @@ ################################################################################ # # Rattail -- Retail Software Framework -# Copyright © 2010-2018 Lance Edgar +# Copyright © 2010-2020 Lance Edgar # # This file is part of Rattail. # @@ -24,18 +24,14 @@ CORE POS -> Rattail data importing """ -from __future__ import unicode_literals, absolute_import - import decimal -import six - -from corepos.db import model as corepos -from corepos.db import Session as CoreSession +from corepos.db.office_op import model as corepos, Session as CoreSession from rattail import importing from rattail.gpc import GPC from rattail.util import OrderedDict +from rattail_corepos import importing as corepos_importing class FromCOREPOSToRattail(importing.FromSQLAlchemyHandler, importing.ToRattailHandler): @@ -66,14 +62,14 @@ class FromCOREPOS(importing.FromSQLAlchemy): """ -class VendorImporter(FromCOREPOS, importing.model.VendorImporter): +class VendorImporter(FromCOREPOS, corepos_importing.model.VendorImporter): """ Importer for vendor data from CORE POS. """ host_model_class = corepos.Vendor - key = 'id' + key = 'corepos_id' supported_fields = [ - 'id', + 'corepos_id', 'name', 'abbreviation', 'special_discount', @@ -82,6 +78,26 @@ class VendorImporter(FromCOREPOS, importing.model.VendorImporter): 'email_address', ] + def cache_query(self): + """ + Return the query to be used when caching "local" data. + """ + # can't just use rattail.db.model b/c the CoreVendor would normally not + # be in there! this still requires custom model to be configured though. + model = self.config.get_model() + + # first get default query + query = super(VendorImporter, self).cache_query() + + # maybe filter a bit, to ensure only "relevant" records are involved + if 'corepos_id' in self.key: + # note, the filter is probably redundant since we INNER JOIN on the + # extension table, and it doesn't allow null ID values. but clarity. + query = query.join(model.CoreVendor)\ + .filter(model.CoreVendor.corepos_id != None) + + return query + def normalize_host_object(self, vendor): special_discount = None @@ -89,9 +105,9 @@ class VendorImporter(FromCOREPOS, importing.model.VendorImporter): special_discount = decimal.Decimal('{:0.3f}'.format(vendor.discount_rate)) return { - 'id': six.text_type(vendor.id), + 'corepos_id': vendor.id, 'name': vendor.name, - 'abbreviation': vendor.abbreviation, + 'abbreviation': vendor.abbreviation or None, 'special_discount': special_discount, 'phone_number': vendor.phone or None, 'fax_number': vendor.fax or None, diff --git a/rattail_corepos/importing/model.py b/rattail_corepos/importing/model.py new file mode 100644 index 0000000..261d026 --- /dev/null +++ b/rattail_corepos/importing/model.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8; -*- +################################################################################ +# +# Rattail -- Retail Software Framework +# Copyright © 2010-2020 Lance Edgar +# +# This file is part of Rattail. +# +# Rattail is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# Rattail is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# Rattail. If not, see . +# +################################################################################ +""" +Rattail model importer extensions, for CORE-POS integration +""" + +from rattail import importing + + +############################## +# core importer overrides +############################## + +class VendorImporter(importing.model.VendorImporter): + + extension_attr = '_corepos' + extension_fields = [ + 'corepos_id', + ]